Merge pull request #800 from flatcar/t-lo/github-actions-add-sdk-update

Trigger build workflow via PR comment instead of PR state; add a workflow to build an updated SDK container,

.github/workflows/ci.yaml

@@ -1,27 +1,38 @@
 name: "Run build"
 on:
-  pull_request:
-    # Run when the PR is opened, reopened, or updated (synchronize)
-    types: [opened, ready_for_review, reopened, synchronize]
   workflow_dispatch:
     inputs:
       image_formats:
+        type: string
         description: |
           Space-separated vendor formats to build.
         required: true
         default: qemu_uefi
+      custom_sdk_version:
+        type: string
+        required: false
+        description: |
+          Custom SDK container version to use for this build.
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} 
-  cancel-in-progress: true
+  workflow_call:
+    inputs:
+      image_formats:
+        type: string
+        description: |
+          Space-separated vendor formats to build.
+        required: true
+        default: qemu_uefi
+      custom_sdk_version:
+        type: string
+        required: false
+        description: |
+          Custom SDK container version to use for this build.
 
 permissions:
   pull-requests: write
 
 jobs:
   packages:
-    # Do not run when still in draft mode but a review was requested anyway
-    if: github.event.pull_request.draft == false
     name: "Build Flatcar packages"
     runs-on:
       - self-hosted
@@ -70,7 +81,6 @@ jobs:
           set -euo pipefail
 
           git checkout ${{ github.event.pull_request.head.sha }}
-          git submodule update
 
       - name: Set environment
         shell: bash
@@ -91,6 +101,10 @@ jobs:
           #  this with its IP address.
           echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV
 
+          if [ -n "${{ github.event.inputs.custom_sdk_version }}" ] ; then
+              echo "CUSTOM_SDK_VERSION=${{ github.event.inputs.custom_sdk_version }}" >> $GITHUB_ENV
+          fi
+
       - name: Build packages
         shell: bash
         run: |
@@ -103,7 +117,7 @@ jobs:
 
           version="alpha-$FLATCAR_VERSION_ID"
           check_version_string "$version"
-          sdk_version="${FLATCAR_SDK_VERSION}"
+          sdk_version="${CUSTOM_SDK_VERSION:-$FLATCAR_SDK_VERSION}"
 
           sdk_name="flatcar-sdk-${arch}"
           docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"

.github/workflows/pr-comment-build-dispatcher.yaml

@@ -0,0 +1,71 @@
+name: "PR command build dispatcher"
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  pull-requests: write
+
+concurrency:
+  group: ${{ github.workflow }}-pr-command-${{ github.head_ref || github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  check_maintainer_membership:
+    # Only run if this is a PR comment that contains a valid command
+    if: |
+      ${{ github.event.issue.pull_request }} &&
+        (    contains(github.event.comment.body, '/update-sdk')
+          || contains(github.event.comment.body, '/build-image') )
+    name: Check if commenter is in the Flatcar maintainers team
+    outputs:
+      maintainers: steps.step1.output.maintainers
+    runs-on:
+      - ubuntu-latest
+    steps:
+      - name: Fetch members of the maintainers team
+        env:
+          requester: ${{ github.event.comment.user.login }}
+        shell: bash
+        run: |
+          set -euo pipefail
+          curl --fail --show-error -L --silent \
+                  -H "Accept: application/vnd.github+json" \
+                  -H "Authorization: Bearer ${{ secrets.GH_ACTIONS_ORG_READ }}" \
+                  -H "X-GitHub-Api-Version: 2022-11-28" \
+                  https://api.github.com/orgs/flatcar/teams/flatcar-maintainers/members \
+              | jq -r '.[].login' > maintainers.txt
+
+          echo "Current members of the maintainers team:"
+          cat maintainers.txt
+
+          res=false
+          echo "Checking for membership of '${{ env.requester }}'"
+          if grep -qE "^${{ env.requester }}$" maintainers.txt ; then
+            echo "Succeeded."
+            res=true
+          else
+            echo "FAILED: '${{ env.requester }} is not a member of the Flatcar maintainers team."
+          fi
+
+          $res
+
+  update_sdk:
+    needs: check_maintainer_membership
+    if: ( always() && needs.check_maintainer_membership.result == 'success'
+                   && contains(github.event.comment.body, '/update-sdk') )
+    name: "Build an updated SDK container"
+    # SDK build needs access to bincache ssh secret
+    secrets: inherit
+    uses: ./.github/workflows/update-sdk.yaml
+
+  build_image:
+    needs: [ check_maintainer_membership, build_sdk ]
+    if: ( always() && needs.check_maintainer_membership.result == 'success'
+                   && (    contains(github.event.comment.body, '/build-image')
+                        || needs.build_sdk.result == 'success' ) )
+    name: "Build the OS image"
+    uses: ./.github/workflows/ci.yaml
+    with:
+      custom_sdk_version: ${{ needs.update_sdk.outputs.sdk_version }}
+      image_formats: qemu_uefi

.github/workflows/update-sdk.yaml

@@ -0,0 +1,145 @@
+name: "Build updated SDK container"
+on:
+  workflow_dispatch:
+    inputs:
+      source_sdk_version:
+        type: string
+        required: false
+        description: |
+          Source SDK container to use. Defaults to version defined in version.txt.
+      custom_sdk_version:
+        type: string
+        required: false
+        description: |
+          Custom SDK container version to build. Defaults to source SDK w/ "-github-[DATE]" appended.
+
+  workflow_call:
+    inputs:
+      source_sdk_version:
+        type: string
+        required: false
+        description: |
+          Source SDK container to use. Defaults to version defined in version.txt.
+      custom_sdk_version:
+        type: string
+        required: false
+        description: |
+          Custom SDK container version to build. Defaults to source SDK w/ "-github-[DATE]" appended, or
+          '-github-pr-[PRNUM]-[DATE]' if the build was triggered from a PR.
+
+permissions:
+  pull-requests: write
+
+jobs:
+  update_sdk:
+    name: "Build an updated SDK container image"
+    runs-on:
+      - self-hosted
+      - debian
+      - build
+      - x64
+    strategy:
+      fail-fast: false
+    outputs:
+      sdk_version: ${{ steps.step4.outputs.sdk_version }}
+    defaults:
+      run:
+        working-directory: scripts
+
+    steps:
+      - name: Prepare machine
+        shell: bash
+        working-directory: ${{ github.workspace }}
+        run: |
+          sudo rm /bin/sh
+          sudo ln -s /bin/bash /bin/sh
+          sudo apt-get install -y ca-certificates curl gnupg lsb-release qemu-user-static git jq openssh-client rsync
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          echo \
+            "deb [signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
+
+      - uses: actions/checkout@v3
+        with:
+          path: scripts
+          fetch-depth: 0
+
+      - name: Set environment
+        shell: bash
+        run: |
+          if [ -n "${{ github.event.inputs.source_sdk_version }}" ] ; then
+              echo "SOURCE_SDK_VERSION=${{ github.event.inputs.source_sdk_version }}" >> $GITHUB_ENV
+          fi
+          if [ -n "${{ github.event.inputs.custom_sdk_version }}" ] ; then
+              echo "CUSTOM_SDK_VERSION=${{ github.event.inputs.custom_sdk_version }}" >> $GITHUB_ENV
+          fi
+
+      - name: Build an updated SDK container
+        shell: bash
+        run: |
+          exec 2>&1
+          set -x
+          set -euo pipefail
+
+          source ci-automation/ci_automation_common.sh
+          source sdk_container/.repo/manifests/version.txt
+
+          version="alpha-$FLATCAR_VERSION_ID"
+          sdk_version="${SOURCE_SDK_VERSION:-$FLATCAR_SDK_VERSION}"
+
+          sdk_name="flatcar-sdk-all"
+          docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")"
+
+          docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}"
+
+          sdk_image="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")"
+
+          # Create version file
+          (
+            source sdk_lib/sdk_container_common.sh
+            create_versionfile "$sdk_version" "$version"
+          )
+
+          if [ -z "${CUSTOM_SDK_VERSION:-}" ] ; then
+              if [ -n "${{ github.event.issue.pull_request }}" ] ; then
+                target_version="${sdk_version}-github-PR-${{ github.event.issue.number }}-$(date '+%Y_%m_%d__%H_%M_%S')"
+              else
+                target_version="${sdk_version}-github-$(date '+%Y_%m_%d__%H_%M_%S')"
+              fi
+          else
+              target_version="${CUSTOM_SDK_VERSION}"
+          fi
+
+          echo "sdk_version=${target_version}" >> "$GITHUB_OUTPUT"
+
+          # This also updates sdk_container/.repo/manifests/version.txt with the new SDK version.
+          ./update_sdk_container_image "${target_version}"
+
+      - name: Upload the SDK container and binary packages to bincache
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          source ci-automation/ci_automation_common.sh
+
+          mkdir -p ~/.ssh
+          trap 'rm -f ~/.ssh/bincache' EXIT
+          echo "${{ secrets.BINCACHESSH }}" > ~/.ssh/bincache
+          chmod 600 ~/.ssh/bincache
+
+          echo "Host ${BUILDCACHE_SERVER}" >> ~/.ssh/config
+          echo "    User ${BUILDCACHE_USER}" >> ~/.ssh/config
+          echo "    IdentityFile ~/.ssh/bincache" >> ~/.ssh/config
+
+          source sdk_container/.repo/manifests/version.txt
+          vernum="${FLATCAR_SDK_VERSION}"
+          docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
+
+          docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-all" "${docker_vernum}"
+          docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-amd64" "${docker_vernum}"
+          docker_image_to_buildcache "${CONTAINER_REGISTRY}/flatcar-sdk-arm64" "${docker_vernum}"
+
+          rm -f ~/.ssh/bincache

ci-automation/garbage_collect.sh

@@ -18,6 +18,8 @@
 # Flatcar CI automation garbage collector.
 #  This script removes development (non-official) build artifacts:
 #   - SDK tarballs, build step containers, and vendor images on buildcache
+#   - SDK containers built via Github actions (e.g. from PRs).
+#      See https://github.com/flatcar/scripts/blob/main/.github/workflows/update-sdk.yaml
 #   - tags from the scripts repository
 #
 #  Garbage collection is based on development (non-official) version tags
@@ -142,6 +144,12 @@ function _garbage_collect_impl() {
         fi
     done
 
+    echo
+    echo "########################################"
+    echo
+    echo    Running cloud garbage collector
+    echo
+
     local mantle_ref
     mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container)
     docker run --pull always --rm --net host \
@@ -153,5 +161,14 @@ function _garbage_collect_impl() {
       --env VMWARE_ESX_CREDS \
       --env OPENSTACK_CREDS \
       -w /work -v "$PWD":/work "${mantle_ref}" /work/ci-automation/garbage_collect_cloud.sh
+
+    echo
+    echo "#############################################"
+    echo
+    echo    Running Github CI SDK garbage collector
+    echo
+
+    source ci-automation/garbage_collect_github_ci_sdk.sh
+    garbage_collect_github_ci
 }
 # --

ci-automation/garbage_collect_github_ci_sdk.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 The Flatcar Maintainers.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
+#
+# garbage_collect_github_ci() should be called after sourcing.
+#
+#  OPTIONAL INPUT
+#  - Number of (recent) Github SDK builds to keep. Defaults to 20.
+#  - DRY_RUN (Env variable). Set to "y" to just list what would be done but not
+#            actually purge anything.
+
+# Flatcar Github CI SDK rebuild automation garbage collector.
+#  This script removes development (non-official) SDK image builds generated via Github CI.
+#
+#  Garbage collection is based on development (non-official) SDK versions listed on
+#     https://bincache.flatcar-linux.net/containers/
+#  and following the pattern [VERSION_NUMBER]*-github-*. The newest 20 builds will be retained,
+#   all older builds will be purged (20 is the default, see OPTIONAL INPUT above).
+
+function garbage_collect_github_ci() {
+    # Run a subshell, so the traps, environment changes and global
+    # variables are not spilled into the caller.
+    (
+        set -euo pipefail
+
+        _garbage_collect_github_ci_impl "${@}"
+    )
+}
+# --
+
+function _garbage_collect_github_ci_impl() {
+    local keep="${1:-20}"
+    local dry_run="${DRY_RUN:-}"
+
+    # Example version string
+    #   <a href="./3598.0.0-nightly-20230508-2100-github-2023_05_09__08_06_54/">
+    #   <a href="./3598.0.0-nightly-20230508-2100-github-pr-12345-2023_05_09__08_06_54/">
+    local versions_detected="$(curl -s https://bincache.flatcar-linux.net/containers/ \
+                | grep -E '\<a href="\./[0-9]+\.[0-9]+.[0-9]+.+-github-.*/">' \
+                | sed 's:.*\"./\([^/]\+\)/".*:\1:' )"
+
+    # Sort versions by date. Since version numbers can differ and this would impact sort, we
+    # 1. insert a "/" between "...-github-[pr-XXX]-" and "[date]..."
+    # 2. sort with delimiter "/" and sorting key 2 (i.e. the date part)
+    # 3. remove the "/"
+   local versions_sorted="$(echo "${versions_detected}" \
+                        | sed 's/\(-github\(-pr-[0-9]*\)*-\)/\1\//' \
+                        | sort -k 2 -t / \
+                        | sed 's:/::')"
+
+    echo "######## Full list of version(s) found ########"
+    echo "${versions_sorted}" | awk '{printf "%5d %s\n", NR, $0}'
+
+    local tail_keep="$((keep + 1))" # for tail -n+...
+    local purge_versions
+    mapfile -t purge_versions < <(tail -n+"${tail_keep}" <<<"${versions_sorted}")
+
+    source ci-automation/ci_automation_common.sh
+    local sshcmd="$(gen_sshcmd)"
+
+    echo
+    echo "######## The following version(s) will be purged ########"
+    if [ "$dry_run" = "y" ] ; then
+        echo
+        echo "(NOTE this is just a dry run since DRY_RUN=y)"
+        echo
+    fi
+    printf '%s\n' "${purge_versions[@]}" | awk -v keep="${keep}" '{if ($0 == "") next; printf "%5d %s\n", NR + keep, $0}'
+    echo
+    echo
+
+    local version=""
+    for version in "${purge_versions[@]}"; do
+        echo "--------------------------------------------"
+        echo
+        echo "#### Processing version '${version}' ####"
+        echo
+
+        local rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${version}/"
+
+        echo "## The following files will be removed ##"
+        $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+            "ls -la ${rmpat} || true"
+
+        if [ "$dry_run" != "y" ] ; then
+            set -x
+            $sshcmd "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}" \
+                "rm -rf ${rmpat}"
+            set +x
+        else
+            echo "## (DRY_RUN=y so not doing anything) ##"
+        fi
+    done
+}
+# --

update_sdk_container_image

@@ -14,6 +14,7 @@ source sdk_lib/sdk_container_common.sh
 
 os_version="$(get_version_from_versionfile)"
 base_sdk_version="$(get_sdk_version_from_versionfile)"
+base_sdk_version="$(vernum_to_docker_image_version "${base_sdk_version}")"
 new_sdk_version=""
 
 keep="false"