From bd600f4e61f80767fb5b8269b726ef2832f8d266 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 26 Sep 2024 14:38:00 +0200 Subject: [PATCH] github --- .github/workflows/pkg-changes-reports.yaml | 149 ++++++++++++++++++ .github/workflows/sync_packages.sh | 60 +++++++ ...ate-portage-stable-packages-from-list.yaml | 57 +------ 3 files changed, 212 insertions(+), 54 deletions(-) create mode 100644 .github/workflows/pkg-changes-reports.yaml create mode 100755 .github/workflows/sync_packages.sh diff --git a/.github/workflows/pkg-changes-reports.yaml b/.github/workflows/pkg-changes-reports.yaml new file mode 100644 index 0000000000..9743d92a67 --- /dev/null +++ b/.github/workflows/pkg-changes-reports.yaml @@ -0,0 +1,149 @@ +name: "Run build" +on: + pull_request: + + workflow_dispatch: + inputs: + base_ref: + type: string + description: Ref or commit SHA to base branch + required: true + default: main + updated_ref: + type: string + description: Ref or commit SHA to updated branch + required: true + default: main + + workflow_call: + inputs: + base_ref: + type: string + description: Ref or commit SHA to base branch + required: true + default: main + updated_ref: + type: string + description: Ref or commit SHA to updated branch + required: true + default: main + +permissions: + pull-requests: write + +jobs: + packages: + name: "Generate package change reports" + runs-on: + - self-hosted + - debian + - build + - x64 + strategy: + fail-fast: false + + steps: + - name: Prepare parameters + id: params + shell: bash + run: | + set -xeuo pipefail + + if [[ ${{ github.event_name }} = 'pull_request' ]]; then + base=${{ github.event.pull_request.base.sha }} + updated=${{ github.event.pull_request.base.sha }} + else + base=${{ inputs.base_ref }} + updated=${{ inputs.updated_ref }} + fi + echo "BASE=${base}" >>"${GITHUB_OUTPUT}" + echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}" + - name: Prepare machine + if: always() && !cancelled() + shell: bash + run: | + set -xeuo pipefail + + sudo apt-get update + sudo apt-get install -y ca-certificates curl git gnupg lsb-release python3 python3-packaging zstd + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + + - name: Checkout pkg-auto scripts + if: always() && !cancelled() + uses: actions/checkout@v4 + with: + ref: main + path: pkg-auto-scripts + fetch-depth: 0 + + - name: Check if automation is in place + if: always() && !cancelled() + id: pkg-auto-in-place + shell: bash + run: | + set -xeuo pipefail + + has_pkg_auto=0 + if [[ -d pkg-auto-scripts/pkg_auto ]]; then has_pkg_auto=1; fi + echo "HAS_PKG_AUTO=${has_pkg_auto}" >>"${GITHUB_OUTPUT}" + + - name: Checkout base scripts + if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled() + uses: actions/checkout@v4 + with: + ref: ${{ steps.params.outputs.BASE }} + path: scripts + fetch-depth: 0 + + - name: Checkout updated scripts + if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled() + uses: actions/checkout@v4 + with: + ref: ${{ steps.params.outputs.UPDATED }} + path: updated-scripts + fetch-depth: 0 + + - name: Prepare aux files + if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled() + shell: bash + run: | + set -xeuo pipefail + pkg-auto-scripts/pkg_auto/download_sdk_and_listings.sh -s scripts -x aux-cleanup aux + + - name: Generate config + if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled() + shell: bash + run: | + set -xeuo pipefail + args=( + -a aux + -n "$(git -C updated-scripts rev-parse)" + -o "$(git -C scripts rev-parse)" + -r reports + -s scripts + ) + pkg-auto-scripts/pkg_auto/generate_config.sh "${args[@]}" config + + - name: Generate reports + if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled() + shell: bash + run: | + set -xeuo pipefail + rc=0 + pkg-auto-scripts/pkg-auto/pkg_auto/generate_reports.sh config || rc=$? + tar --zstd -cf reports.tar.zst reports + + - name: Upload reports + if: steps.pkg-auto-in-place.outputs.HAS_PKG_AUTO == 1 && !cancelled() + uses: actions/upload-artifact@v4 + with: + retention-days: 7 + name: reports + path: | + reports.tar.zst diff --git a/.github/workflows/sync_packages.sh b/.github/workflows/sync_packages.sh new file mode 100755 index 0000000000..0fed10c095 --- /dev/null +++ b/.github/workflows/sync_packages.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPTS_REPO=${1}; shift +SCRIPTS_BASE_BRANCH=${1}; shift +GENTOO_REPO=${1}; shift + +today_date=$(date +%Y-%m-%d) +branch_name=buildbot/weekly-portage-stable-package-updates-${today_date} +"${SCRIPTS_REPO}/pkg_auto/generate_config.sh" -o "${SCRIPTS_BASE_BRANCH}" -r reports -s scripts -x trap config +"${SCRIPTS_REPO}/pkg_auto/sync_packages.sh" -w wd config "${branch_name}" "${GENTOO_REPO}" +old_head=$(git -C scripts rev-parse "${SCRIPTS_BASE_BRANCH}") +new_head=$(git -C scripts rev-parse "${branch_name}") + +if [[ ${new_head} == "${old_head}" ]]; then + echo 'UPDATED=0' >>"${GITHUB_OUTPUT}" + exit 0 +fi + +body_file=./pr_body +cat <"${body_file}" +CI: TODO + +-- + +TODO: Changes. + +-- +EOF + +shopt -s nullglob + +for report in reports/*; do + if [[ ! -f ${report} ]]; then + continue + fi + name=${report#reports/} + cat <>"${body_file}" + +from ${name@Q}: + +``` +$(cat "${report}") +``` + +-- +EOF +done + +cat <>"${body_file}" + +- [ ] changelog +- [ ] image diff +EOF + +echo "UPDATED=1" >>"${GITHUB_OUTPUT}" +echo "TODAY_DATE=${today_date}" >>"${GITHUB_OUTPUT}" +echo "BRANCH=${branch_name}" >>"${GITHUB_OUTPUT}" +echo "BODY_PATH=${body_file}" >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/update-portage-stable-packages-from-list.yaml b/.github/workflows/update-portage-stable-packages-from-list.yaml index 044b77af86..c7b89b17be 100644 --- a/.github/workflows/update-portage-stable-packages-from-list.yaml +++ b/.github/workflows/update-portage-stable-packages-from-list.yaml @@ -24,69 +24,18 @@ jobs: # that made the changes to the package. fetch-depth: 250000 ref: master - - name: Check out build scripts - uses: actions/checkout@v4 - with: - repository: flatcar/flatcar-build-scripts - path: flatcar-build-scripts - name: Update listed packages id: update-listed-packages - run: | - git config --global user.name "Flatcar Buildbot" - git config --global user.email "buildbot@flatcar-linux.org" - old_head=$(git -C scripts rev-parse HEAD) - packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list) - gentoo_repo=$(realpath gentoo) - build_scripts=$(realpath flatcar-build-scripts) - pushd scripts/sdk_container/src/third_party/portage-stable - while read -r package; do - if [[ ! -e "${package}" ]]; then - # If this happens, it means that the package was moved to overlay - # or dropped, the list ought to be updated. - echo "::warning title=${package}::Nonexistent package" - continue - fi - if [[ ! -e "${gentoo_repo}/${package}" ]]; then - # If this happens, it means that the package was obsoleted or moved - # in Gentoo. The obsoletion needs to be handled in the case-by-case - # manner, while move should be handled by doing the same move - # in portage-stable. The build should not break because of the move, - # because most likely it's already reflected in the profiles/updates - # directory. - echo "::warning title=${package}::Obsolete or moved package" - continue - fi - GENTOO_REPO="${gentoo_repo}" "${build_scripts}/sync-with-gentoo" "${package}" - done < <(grep '^[^#]' "${packages_list}") - popd - new_head=$(git -C scripts rev-parse HEAD) - updated=0 - if [[ "${new_head}" != "${old_head}" ]]; then - updated=1 - fi - todaydate=$(date +%Y-%m-%d) - echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}" - echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}" + run: ./scripts/.github/workflows/sync_packages.sh ./scripts main ./gentoo - name: Create pull request for main branch uses: peter-evans/create-pull-request@v6 if: steps.update-listed-packages.outputs.UPDATED == 1 with: token: ${{ secrets.GITHUB_TOKEN }} path: scripts - branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }} - delete-branch: true + branch: ${{steps.update-listed-packages.outputs.BRANCH }} base: main title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }} - body: | - CI: TODO - - -- - - TODO: Changes. - - -- - - - [ ] changelog - - [ ] image diff + body-path: ${{steps.update-listed-packages.outputs.BODY_PATH }} labels: main draft: true