From 0513a6e82c007ae639bb28727c779371e9be5cfd Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 29 May 2023 16:46:12 +0300 Subject: [PATCH 1/6] enable on PR Signed-off-by: Gabriel Adrian Samfira --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3acd44b646..42ff1d0e0f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,5 +1,6 @@ name: "Run build" on: + pull_request: workflow_dispatch: inputs: image_formats: @@ -29,7 +30,7 @@ on: Custom SDK container version to use for this build. permissions: - pull-requests: write + pull-requests: write jobs: packages: @@ -39,6 +40,7 @@ jobs: - debian - build - x64 + environment: main strategy: fail-fast: false matrix: From 215fb8f823e4d7375ee75a5d87401cf58afd95fc Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 30 May 2023 21:41:21 +0300 Subject: [PATCH 2/6] Refactor workflows Signed-off-by: Gabriel Adrian Samfira --- .github/workflows/ci.yaml | 4 +- .../pr-comment-build-dispatcher.yaml | 40 +++++++++---- .github/workflows/pr-workflows.yaml | 56 +++++++++++++++++++ 3 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/pr-workflows.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 42ff1d0e0f..3acd44b646 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,5 @@ name: "Run build" on: - pull_request: workflow_dispatch: inputs: image_formats: @@ -30,7 +29,7 @@ on: Custom SDK container version to use for this build. permissions: - pull-requests: write + pull-requests: write jobs: packages: @@ -40,7 +39,6 @@ jobs: - debian - build - x64 - environment: main strategy: fail-fast: false matrix: diff --git a/.github/workflows/pr-comment-build-dispatcher.yaml b/.github/workflows/pr-comment-build-dispatcher.yaml index 4d334d2601..f6396973b6 100644 --- a/.github/workflows/pr-comment-build-dispatcher.yaml +++ b/.github/workflows/pr-comment-build-dispatcher.yaml @@ -11,18 +11,18 @@ concurrency: cancel-in-progress: true jobs: - check_maintainer_membership: + run_pre_checks: # Only run if this is a PR comment that contains a valid command - if: | - ${{ github.event.issue.pull_request }} && - ( contains(github.event.comment.body, '/update-sdk') || contains(github.event.comment.body, '/build-image') ) + if: ${{ github.event.issue.pull_request }} && contains(github.event.comment.body, '/build-image') name: Check if commenter is in the Flatcar maintainers team outputs: maintainers: steps.step1.output.maintainers + sdk_changes: steps.step3.outputs.sdk_changes runs-on: - ubuntu-latest steps: - name: Fetch members of the maintainers team + id: step1 env: requester: ${{ github.event.comment.user.login }} shell: bash @@ -49,25 +49,45 @@ jobs: $res + - uses: actions/checkout@v3 + id: step2 + with: + path: scripts + fetch-depth: 0 + + - uses: dorny/paths-filter@v2 + id: filter + with: + working-directory: scripts + filters: | + sdk_changes: + - 'sdk_container/**' + - 'sdk_libs/**' + + - name: Set outputs + id: step3 + shell: bash + run: | + echo "sdk_changes=${{ steps.filter.outputs.sdk_changes }}" >> $GITHUB_OUTPUT + - name: Post a link to the workflow run to the PR + id: step4 uses: mshick/add-pr-comment@v2 with: issue: ${{ github.event.issue.pull_request.number }} message: "Build action triggered: [${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" update_sdk: - needs: check_maintainer_membership - if: ( needs.check_maintainer_membership.result == 'success' - && contains(github.event.comment.body, '/update-sdk') ) + needs: run_pre_checks + if: needs.run_pre_checks.result == 'success' && needs.run_pre_checks.outputs.sdk_changes == 'true' name: "Build an updated SDK container" # SDK build needs access to bincache ssh secret secrets: inherit uses: ./.github/workflows/update-sdk.yaml build_image: - needs: [ check_maintainer_membership, update_sdk ] - if: ( needs.check_maintainer_membership.result == 'success' - && ( contains(github.event.comment.body, '/build-image') || needs.update_sdk.result == 'success' ) ) + needs: [ run_pre_checks, update_sdk ] + if: (always() && ! cancelled()) && needs.run_pre_checks.result == 'success' && contains(github.event.comment.body, '/build-image') name: "Build the OS image" uses: ./.github/workflows/ci.yaml with: diff --git a/.github/workflows/pr-workflows.yaml b/.github/workflows/pr-workflows.yaml new file mode 100644 index 0000000000..6d0d32305c --- /dev/null +++ b/.github/workflows/pr-workflows.yaml @@ -0,0 +1,56 @@ +name: "Run PR workflows" +on: + pull_request: + +permissions: + pull-requests: write + +concurrency: + group: ${{ github.workflow }}-pr-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + check_for_sdk_changes: + name: "Check for SDK changes" + runs-on: ubuntu-latest + environment: development + outputs: + sdk_changes: ${{ steps.step2.outputs.sdk_changes }} + steps: + - uses: actions/checkout@v3 + id: step1 + with: + path: scripts + fetch-depth: 0 + + - uses: dorny/paths-filter@v2 + id: filter + with: + working-directory: scripts + filters: | + sdk_changes: + - 'sdk_container/**' + - 'sdk_libs/**' + + - name: Set outputs + id: step2 + shell: bash + run: | + echo "sdk_changes=${{ steps.filter.outputs.sdk_changes }}" >> $GITHUB_OUTPUT + + update_sdk: + name: "Build an updated SDK container" + needs: [ check_for_sdk_changes ] + if: needs.check_for_sdk_changes.sdk_changes == 'true' + # SDK build needs access to bincache ssh secret + secrets: inherit + uses: ./.github/workflows/update-sdk.yaml + + build_image: + needs: [ update_sdk ] + if: (always() && ! cancelled()) && needs.update_sdk.result != 'failure' + name: "Build the OS image" + uses: ./.github/workflows/ci.yaml + with: + custom_sdk_version: ${{ needs.update_sdk.outputs.sdk_version }} + image_formats: qemu_uefi From 66bfe6a6e5e44248a17468c36f1cf4284f97cf29 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Wed, 31 May 2023 16:09:23 +0300 Subject: [PATCH 3/6] Look for /update-sdk in PR description Signed-off-by: Gabriel Adrian Samfira --- .../pr-comment-build-dispatcher.yaml | 27 +++----------- .github/workflows/pr-workflows.yaml | 37 ++++++++----------- 2 files changed, 21 insertions(+), 43 deletions(-) diff --git a/.github/workflows/pr-comment-build-dispatcher.yaml b/.github/workflows/pr-comment-build-dispatcher.yaml index f6396973b6..4110d351bd 100644 --- a/.github/workflows/pr-comment-build-dispatcher.yaml +++ b/.github/workflows/pr-comment-build-dispatcher.yaml @@ -13,11 +13,11 @@ concurrency: jobs: run_pre_checks: # Only run if this is a PR comment that contains a valid command - if: ${{ github.event.issue.pull_request }} && contains(github.event.comment.body, '/build-image') + if: ${{ github.event.issue.pull_request }} && ( contains(github.event.comment.body, '/build-image') || contains(github.event.comment.body, '/update-sdk')) name: Check if commenter is in the Flatcar maintainers team outputs: maintainers: steps.step1.output.maintainers - sdk_changes: steps.step3.outputs.sdk_changes + sdk_changes: ${{ steps.step3.outputs.sdk_changes }} runs-on: - ubuntu-latest steps: @@ -49,29 +49,14 @@ jobs: $res - - uses: actions/checkout@v3 - id: step2 - with: - path: scripts - fetch-depth: 0 - - - uses: dorny/paths-filter@v2 - id: filter - with: - working-directory: scripts - filters: | - sdk_changes: - - 'sdk_container/**' - - 'sdk_libs/**' - - name: Set outputs - id: step3 + id: step2 shell: bash run: | - echo "sdk_changes=${{ steps.filter.outputs.sdk_changes }}" >> $GITHUB_OUTPUT + echo "sdk_changes=${{ contains(github.event.comment.body, '/update-sdk') }}" >> $GITHUB_OUTPUT - name: Post a link to the workflow run to the PR - id: step4 + id: step3 uses: mshick/add-pr-comment@v2 with: issue: ${{ github.event.issue.pull_request.number }} @@ -87,7 +72,7 @@ jobs: build_image: needs: [ run_pre_checks, update_sdk ] - if: (always() && ! cancelled()) && needs.run_pre_checks.result == 'success' && contains(github.event.comment.body, '/build-image') + if: (always() && ! cancelled()) && needs.run_pre_checks.result == 'success' && needs.update_sdk.result != 'failure' && contains(github.event.comment.body, '/build-image') name: "Build the OS image" uses: ./.github/workflows/ci.yaml with: diff --git a/.github/workflows/pr-workflows.yaml b/.github/workflows/pr-workflows.yaml index 6d0d32305c..6a8302f3e2 100644 --- a/.github/workflows/pr-workflows.yaml +++ b/.github/workflows/pr-workflows.yaml @@ -10,44 +10,37 @@ concurrency: cancel-in-progress: true jobs: - check_for_sdk_changes: - name: "Check for SDK changes" + pre_check: + name: "Check if we need to update the SDK" runs-on: ubuntu-latest + # Setting the environment is the more important reason we need this job. + # We use this job as a gate, so we can approve the PR workflow only once. If + # we set this in the update_sdk job and in the build_image job, we would have + # to approve the workflow for every job that kicks off. Given that the jobs + # are sequenced, this is cumbersome. Use this job as a gate and make the rest + # dependent on it. environment: development outputs: - sdk_changes: ${{ steps.step2.outputs.sdk_changes }} + sdk_changes: ${{ steps.step1.outputs.sdk_changes }} steps: - - uses: actions/checkout@v3 - id: step1 - with: - path: scripts - fetch-depth: 0 - - - uses: dorny/paths-filter@v2 - id: filter - with: - working-directory: scripts - filters: | - sdk_changes: - - 'sdk_container/**' - - 'sdk_libs/**' - - name: Set outputs - id: step2 + id: step1 shell: bash run: | - echo "sdk_changes=${{ steps.filter.outputs.sdk_changes }}" >> $GITHUB_OUTPUT + echo "sdk_changes=${{ contains(github.event.pull_request.body, '/update-sdk') }}" >> $GITHUB_OUTPUT update_sdk: name: "Build an updated SDK container" - needs: [ check_for_sdk_changes ] - if: needs.check_for_sdk_changes.sdk_changes == 'true' + needs: [ pre_check ] + if: needs.pre_check.outputs.sdk_changes == 'true' # SDK build needs access to bincache ssh secret secrets: inherit uses: ./.github/workflows/update-sdk.yaml build_image: needs: [ update_sdk ] + # The update-sdk job may be skipped, which is fine. We only care if it tried to + # run, but failed. if: (always() && ! cancelled()) && needs.update_sdk.result != 'failure' name: "Build the OS image" uses: ./.github/workflows/ci.yaml From 91b96b21b331aba06bd392303e13d45846f36e56 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 1 Jun 2023 17:40:31 +0300 Subject: [PATCH 4/6] Define output for update_sdk Signed-off-by: Gabriel Adrian Samfira --- .github/workflows/pr-workflows.yaml | 2 +- .github/workflows/update-sdk.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr-workflows.yaml b/.github/workflows/pr-workflows.yaml index 6a8302f3e2..b83f178183 100644 --- a/.github/workflows/pr-workflows.yaml +++ b/.github/workflows/pr-workflows.yaml @@ -41,7 +41,7 @@ jobs: needs: [ update_sdk ] # The update-sdk job may be skipped, which is fine. We only care if it tried to # run, but failed. - if: (always() && ! cancelled()) && needs.update_sdk.result != 'failure' + if: (always() && !cancelled()) && needs.update_sdk.result != 'failure' name: "Build the OS image" uses: ./.github/workflows/ci.yaml with: diff --git a/.github/workflows/update-sdk.yaml b/.github/workflows/update-sdk.yaml index c25c46e4a5..3465c86fad 100644 --- a/.github/workflows/update-sdk.yaml +++ b/.github/workflows/update-sdk.yaml @@ -14,6 +14,10 @@ on: Custom SDK container version to build. Defaults to source SDK w/ "-github-[DATE]" appended. workflow_call: + outputs: + sdk_version: + description: "The version of the SDK container that was built" + value: ${{ jobs.update_sdk.outputs.sdk_version }} inputs: source_sdk_version: type: string From 3a0a28e32a24088d16e25f8888981b6cd156cd10 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 1 Jun 2023 17:44:17 +0300 Subject: [PATCH 5/6] Make jobs cancelable Signed-off-by: Gabriel Adrian Samfira --- .github/workflows/ci.yaml | 4 ++-- .github/workflows/run-kola-tests.yaml | 6 +++--- .github/workflows/update-sdk.yaml | 1 + 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3acd44b646..9a23201197 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -147,7 +147,7 @@ jobs: -cvf binpkgs.tar . - name: Extract build logs - if: always() + if: always() && !cancelled() shell: bash run: | set -euo pipefail @@ -158,7 +158,7 @@ jobs: /build/${arch}-usr/var/tmp/portage - name: Upload build logs - if: always() + if: always() && !cancelled() uses: actions/upload-artifact@v3 with: retention-days: 7 diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 8f4234a5e7..35d624b366 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -243,7 +243,7 @@ jobs: set -e - name: Upload detailed test logs - if: always() + if: always() && !cancelled() uses: actions/upload-artifact@v3 with: name: ${{ matrix.arch }}-test-logs-and-results @@ -255,7 +255,7 @@ jobs: scripts/results-*.md - name: Upload raw TAP files of all runs for later merging - if: always() + if: always() && !cancelled() uses: actions/upload-artifact@v3 with: name: ${{ matrix.arch }}-raw-tapfiles @@ -266,7 +266,7 @@ jobs: merge_and_publish_results: name: "Merge TAP reports and post results" needs: tests - if: always() + if: always() && !cancelled() runs-on: - self-hosted - debian diff --git a/.github/workflows/update-sdk.yaml b/.github/workflows/update-sdk.yaml index 3465c86fad..8c039463a6 100644 --- a/.github/workflows/update-sdk.yaml +++ b/.github/workflows/update-sdk.yaml @@ -121,6 +121,7 @@ jobs: target_version="${CUSTOM_SDK_VERSION}" fi + echo "setting sdk_version=${target_version} as a github output" echo "sdk_version=${target_version}" >> "$GITHUB_OUTPUT" # This also updates sdk_container/.repo/manifests/version.txt with the new SDK version. From 6819fd1a9e37a6e6791dfbca52a2e0408f4f1754 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 1 Jun 2023 19:18:23 +0300 Subject: [PATCH 6/6] Properly reference inputs Signed-off-by: Gabriel Adrian Samfira --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9a23201197..53289a9fa2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -89,7 +89,7 @@ jobs: echo "arch=${arch}" >> $GITHUB_ENV IMAGE_FORMATS="qemu_uefi" - [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}" + [ -z "${{ inputs.image_formats }}" ] || IMAGE_FORMATS="${{ inputs.image_formats }}" echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV # Artifact root for images and torcx tarball as seen from within the container @@ -101,8 +101,8 @@ jobs: # this with its IP address. echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV - if [ -n "${{ github.event.inputs.custom_sdk_version }}" ] ; then - echo "CUSTOM_SDK_VERSION=${{ github.event.inputs.custom_sdk_version }}" >> $GITHUB_ENV + if [ -n "${{ inputs.custom_sdk_version }}" ] ; then + echo "CUSTOM_SDK_VERSION=${{ inputs.custom_sdk_version }}" >> $GITHUB_ENV fi - name: Build packages