github

.github/workflows/pkg-changes-reports.yaml

@@ -0,0 +1,149 @@
+name: "Run build"
+on:
+  pull_request:
+
+  workflow_dispatch:
+    inputs:
+      base_ref:
+        type: string
+        description: Ref or commit SHA to base branch
+        required: true
+        default: main
+      updated_ref:
+        type: string
+        description: Ref or commit SHA to updated branch
+        required: true
+        default: main
+
+  workflow_call:
+    inputs:
+      base_ref:
+        type: string
+        description: Ref or commit SHA to base branch
+        required: true
+        default: main
+      updated_ref:
+        type: string
+        description: Ref or commit SHA to updated branch
+        required: true
+        default: main
+
+permissions:
+  pull-requests: write
+
+jobs:
+  packages:
+    name: "Generate package change reports"
+    runs-on:
+      - self-hosted
+      - debian
+      - build
+      - x64
+    strategy:
+      fail-fast: false
+
+    steps:
+      - name: Prepare parameters
+        id: params
+        shell: bash
+        run: |
+          set -xeuo pipefail
+
+          if [[ ${{ github.event_name }} = 'pull_request' ]]; then
+            base=${{ github.event.pull_request.base.sha }}
+            updated=${{ github.event.pull_request.base.sha }}
+          else
+            base=${{ inputs.base_ref }}
+            updated=${{ inputs.updated_ref }}
+          fi
+          echo "BASE=${base}" >>"${GITHUB_OUTPUT}"
+          echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}"
+      - name: Prepare machine
+        if: always() && !cancelled()
+        shell: bash
+        run: |
+          set -xeuo pipefail
+
+          sudo apt-get update
+          sudo apt-get install -y ca-certificates curl git gnupg lsb-release python3 python3-packaging zstd
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+          echo \
+            "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+            $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get update
+          sudo apt-get install -y docker-ce docker-ce-cli containerd.io
+
+      - name: Checkout pkg-auto scripts
+        if: always() && !cancelled()
+        uses: actions/checkout@v4
+        with:
+          ref: main
+          path: pkg-auto-scripts
+          fetch-depth: 0
+
+      - name: Check if automation is in place
+        if: always() && !cancelled()
+        id: pkg-auto-in-place
+        shell: bash
+        run: |
+          set -xeuo pipefail
+
+          has_pkg_auto=0
+          if [[ -d pkg-auto-scripts/pkg_auto ]]; then has_pkg_auto=1; fi
+          echo "HAS_PKG_AUTO=${has_pkg_auto}" >>"${GITHUB_OUTPUT}"
+
+      - name: Checkout base scripts
+        if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled()
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.params.outputs.BASE }}
+          path: scripts
+          fetch-depth: 0
+
+      - name: Checkout updated scripts
+        if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled()
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ steps.params.outputs.UPDATED }}
+          path: updated-scripts
+          fetch-depth: 0
+
+      - name: Prepare aux files
+        if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled()
+        shell: bash
+        run: |
+          set -xeuo pipefail
+          pkg-auto-scripts/pkg_auto/download_sdk_and_listings.sh -s scripts -x aux-cleanup aux
+
+      - name: Generate config
+        if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled()
+        shell: bash
+        run: |
+          set -xeuo pipefail
+          args=(
+            -a aux
+            -n "$(git -C updated-scripts rev-parse)"
+            -o "$(git -C scripts rev-parse)"
+            -r reports
+            -s scripts
+          )
+          pkg-auto-scripts/pkg_auto/generate_config.sh "${args[@]}" config
+
+      - name: Generate reports
+        if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled()
+        shell: bash
+        run: |
+          set -xeuo pipefail
+          rc=0
+          pkg-auto-scripts/pkg-auto/pkg_auto/generate_reports.sh config || rc=$?
+          tar --zstd -cf reports.tar.zst reports
+
+      - name: Upload reports
+        if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled()
+        uses: actions/upload-artifact@v4
+        with:
+          retention-days: 7
+          name: reports
+          path: |
+            reports.tar.zst

.github/workflows/sync_packages.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPTS_REPO=${1}; shift
+SCRIPTS_BASE_BRANCH=${1}; shift
+GENTOO_REPO=${1}; shift
+
+today_date=$(date +%Y-%m-%d)
+branch_name=buildbot/weekly-portage-stable-package-updates-${today_date}
+"${SCRIPTS_REPO}/pkg_auto/generate_config.sh" -o "${SCRIPTS_BASE_BRANCH}" -r reports -s scripts -x trap config
+"${SCRIPTS_REPO}/pkg_auto/sync_packages.sh" -w wd config "${branch_name}" "${GENTOO_REPO}"
+old_head=$(git -C scripts rev-parse "${SCRIPTS_BASE_BRANCH}")
+new_head=$(git -C scripts rev-parse "${branch_name}")
+
+if [[ ${new_head} == "${old_head}" ]]; then
+    echo 'UPDATED=0' >>"${GITHUB_OUTPUT}"
+    exit 0
+fi
+
+body_file=./pr_body
+cat <<EOF >"${body_file}"
+CI: TODO
+
+--
+
+TODO: Changes.
+
+--
+EOF
+
+shopt -s nullglob
+
+for report in reports/*; do
+    if [[ ! -f ${report} ]]; then
+        continue
+    fi
+    name=${report#reports/}
+    cat <<EOF >>"${body_file}"
+
+from ${name@Q}:
+
+```
+$(cat "${report}")
+```
+
+--
+EOF
+done
+
+cat <<EOF >>"${body_file}"
+
+- [ ] changelog
+- [ ] image diff
+EOF
+
+echo "UPDATED=1" >>"${GITHUB_OUTPUT}"
+echo "TODAY_DATE=${today_date}" >>"${GITHUB_OUTPUT}"
+echo "BRANCH=${branch_name}" >>"${GITHUB_OUTPUT}"
+echo "BODY_PATH=${body_file}" >>"${GITHUB_OUTPUT}"

.github/workflows/update-portage-stable-packages-from-list.yaml

@@ -24,69 +24,18 @@ jobs:
           # that made the changes to the package.
           fetch-depth: 250000
           ref: master
-      - name: Check out build scripts
-        uses: actions/checkout@v4
-        with:
-          repository: flatcar/flatcar-build-scripts
-          path: flatcar-build-scripts
       - name: Update listed packages
         id: update-listed-packages
-        run: |
-          git config --global user.name "Flatcar Buildbot"
-          git config --global user.email "buildbot@flatcar-linux.org"
-          old_head=$(git -C scripts rev-parse HEAD)
-          packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list)
-          gentoo_repo=$(realpath gentoo)
-          build_scripts=$(realpath flatcar-build-scripts)
-          pushd scripts/sdk_container/src/third_party/portage-stable
-          while read -r package; do
-              if [[ ! -e "${package}" ]]; then
-                  # If this happens, it means that the package was moved to overlay
-                  # or dropped, the list ought to be updated.
-                  echo "::warning title=${package}::Nonexistent package"
-                  continue
-              fi
-              if [[ ! -e "${gentoo_repo}/${package}" ]]; then
-                  # If this happens, it means that the package was obsoleted or moved
-                  # in Gentoo. The obsoletion needs to be handled in the case-by-case
-                  # manner, while move should be handled by doing the same move
-                  # in portage-stable. The build should not break because of the move,
-                  # because most likely it's already reflected in the profiles/updates
-                  # directory.
-                  echo "::warning title=${package}::Obsolete or moved package"
-                  continue
-              fi
-              GENTOO_REPO="${gentoo_repo}" "${build_scripts}/sync-with-gentoo" "${package}"
-          done < <(grep '^[^#]' "${packages_list}")
-          popd
-          new_head=$(git -C scripts rev-parse HEAD)
-          updated=0
-          if [[ "${new_head}" != "${old_head}" ]]; then
-              updated=1
-          fi
-          todaydate=$(date +%Y-%m-%d)
-          echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}"
-          echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}"
+        run: ./scripts/.github/workflows/sync_packages.sh ./scripts main ./gentoo
       - name: Create pull request for main branch
         uses: peter-evans/create-pull-request@v6
         if: steps.update-listed-packages.outputs.UPDATED == 1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           path: scripts
-          branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }}
-          delete-branch: true
+          branch: ${{steps.update-listed-packages.outputs.BRANCH }}
           base: main
           title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }}
-          body: |
-            CI: TODO
-
-            --
-
-            TODO: Changes.
-
-            --
-
-            - [ ] changelog
-            - [ ] image diff
+          body-path: ${{steps.update-listed-packages.outputs.BODY_PATH }}
           labels: main
           draft: true