diff --git a/.github/workflows/cacerts-apply-patch.sh b/.github/workflows/cacerts-apply-patch.sh new file mode 100755 index 0000000000..032e92af1d --- /dev/null +++ b/.github/workflows/cacerts-apply-patch.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +pushd "${SDK_OUTER_OVERLAY}" + +# Parse the Manifest file for already present source files and keep the latest version in the current series +VERSION_OLD=$(sed -n "s/^DIST nss-\([0-9]*\.[0-9]*\).*$/\1/p" app-misc/ca-certificates/Manifest | sort -ruV | head -n1) +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest ca-certificates, nothing to do" + exit 0 +fi + +EBUILD_FILENAME=$(get_ebuild_filename app-misc/ca-certificates "${VERSION_OLD}") +git mv "${EBUILD_FILENAME}" "app-misc/ca-certificates/ca-certificates-${VERSION_NEW}.ebuild" + +popd + +URLVERSION=$(echo "${VERSION_NEW}" | tr '.' '_') +URL="https://firefox-source-docs.mozilla.org/security/nss/releases/nss_${URLVERSION}.html" + +generate_update_changelog 'ca-certificates' "${VERSION_NEW}" "${URL}" 'ca-certificates' + +commit_changes app-misc/ca-certificates "${VERSION_OLD}" "${VERSION_NEW}" + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/cacerts-release.yaml b/.github/workflows/cacerts-release.yaml new file mode 100644 index 0000000000..2eac2121e9 --- /dev/null +++ b/.github/workflows/cacerts-release.yaml @@ -0,0 +1,67 @@ +name: Get the latest ca-certificates release for all maintained branches +on: + schedule: + - cron: '0 7 * * 1' + workflow_dispatch: + +jobs: + get-cacerts-release: + strategy: + matrix: + channel: [main,alpha,beta,stable,lts,lts-old] + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Check out main scripts branch for GitHub workflow scripts only + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: gha + ref: main + - name: Figure out branch + id: figure-out-branch + run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}' + - name: Check out work scripts branch for updating + if: steps.figure-out-branch.outputs.SKIP == 0 + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: work + ref: ${{ steps.figure-out-branch.outputs.BRANCH }} + - name: Figure out latest ca-certificates release version + if: steps.figure-out-branch.outputs.SKIP == 0 + id: nss-latest-release + run: | + nssVersion=$(git ls-remote --tags https://github.com/nss-dev/nss | cut -f2 | sed -n "s/refs\/tags\/NSS_\([0-9]_[0-9_]*\).*_RTM$/\1/p" | sort -s -t_ -k1,1 -k2,2n -k3,3n | tr '_' '.' | tail -n1) + echo "NSS_VERSION=${nssVersion}" >>"${GITHUB_OUTPUT}" + - name: Set up Flatcar SDK + if: steps.figure-out-branch.outputs.SKIP == 0 + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/work" + CHANNEL: ${{ steps.figure-out-branch.outputs.LABEL }} + # This will be empty for the main channel, but we handle + # this case inside setup-flatcar-sdk.sh. + MIRROR_LINK: ${{ steps.figure-out-branch.outputs.LINK }} + run: gha/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch + if: steps.figure-out-branch.outputs.SKIP == 0 + id: apply-patch + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/gha" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/work" + VERSION_NEW: ${{ steps.nss-latest-release.outputs.NSS_VERSION }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: gha/.github/workflows/cacerts-apply-patch.sh + - name: Create pull request + if: (steps.figure-out-branch.outputs.SKIP == 0) && (steps.apply-patch.outputs.UPDATE_NEEDED == 1) + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: work + branch: "cacerts-${{ steps.nss-latest-release.outputs.NSS_VERSION }}-${{ steps.figure-out-branch.outputs.BRANCH }}" + base: ${{ steps.figure-out-branch.outputs.BRANCH }} + title: Update ca-certificates in ${{ steps.figure-out-branch.outputs.BRANCH }} from ${{ steps.apply-patch.outputs.VERSION_OLD }} to ${{ steps.nss-latest-release.outputs.NSS_VERSION }} + body: Subject says it all. + labels: ${{ steps.figure-out-branch.outputs.LABEL }} diff --git a/.github/workflows/common.sh b/.github/workflows/common.sh new file mode 100644 index 0000000000..faed205407 --- /dev/null +++ b/.github/workflows/common.sh @@ -0,0 +1,186 @@ +#!/bin/bash + +function fail() { + echo "$*" >/dev/stderr + exit 1 +} + +if [[ -z "${WORK_SCRIPTS_DIR:-}" ]]; then + fail "WORK_SCRIPTS_DIR env var unset. It should point to the scripts repo which will be updated." +fi + +if [[ ! -d "${WORK_SCRIPTS_DIR:-}" ]]; then + fail "WORK_SCRIPTS_DIR env var does not point to a directory. It should point to the scripts repo which will be updated." +fi + +readonly SDK_OUTER_TOPDIR="${WORK_SCRIPTS_DIR}" +readonly SDK_OUTER_OVERLAY="${SDK_OUTER_TOPDIR}/sdk_container/src/third_party/coreos-overlay" +readonly SDK_INNER_SRCDIR="/mnt/host/source/src" +readonly SDK_INNER_OVERLAY="${SDK_INNER_SRCDIR}/third_party/coreos-overlay" + +readonly BUILDBOT_USERNAME="Flatcar Buildbot" +readonly BUILDBOT_USEREMAIL="buildbot@flatcar-linux.org" + +# This enters the SDK container and executes the passed commands +# inside it. Requires PACKAGES_CONTAINER and SDK_NAME to be defined. +function enter() { + if [[ -z "${PACKAGES_CONTAINER}" ]]; then + fail "PACKAGES_CONTAINER env var unset. It should contain the name of the SDK container." + fi + if [[ -z "${SDK_NAME}" ]]; then + fail "SDK_NAME env var unset. It should contain the name of the SDK docker image." + fi + "${SDK_OUTER_TOPDIR}/run_sdk_container" \ + -n "${PACKAGES_CONTAINER}" \ + -C "${SDK_NAME}" \ + "${@}" +} + +# Return a valid ebuild file name for ebuilds of the given category name, +# package name, and the old version. If the single ebuild file already exists, +# then simply return that. If the file does not exist, then we should fall back +# to a similar file including $VERSION_OLD. +# For example, if VERSION_OLD == 1.0 and 1.0.ebuild does not exist, but only +# 1.0-r1.ebuild is there, then we figure out its most similar valid name by +# running "ls -1 ...*.ebuild | sort -ruV | head -n1". +function get_ebuild_filename() { + local pkg="${1}"; shift + local version="${1}"; shift + local name="${pkg##*/}" + local ebuild_basename="${pkg}/${name}-${version}" + + if [[ ! -d "${pkg}" ]]; then + fail "No such package in '${PWD}': '${pkg}'" + fi + if [ -f "${ebuild_basename}.ebuild" ]; then + echo "${ebuild_basename}.ebuild" + else + ls -1 "${ebuild_basename}"*.ebuild | sort --reverse --unique --version-sort | head --lines 1 + fi +} + +function prepare_git_repo() { + git -C "${SDK_OUTER_TOPDIR}" config user.name "${BUILDBOT_USERNAME}" + git -C "${SDK_OUTER_TOPDIR}" config user.email "${BUILDBOT_USEREMAIL}" +} + +# Regenerates a manifest file using an ebuild of a given package with +# a given version. +# +# Example: +# regenerate_manifest dev-lang/go 1.20.2 +function regenerate_manifest() { + local pkg="${1}"; shift + local version="${1}"; shift + local name="${pkg##*/}" + local ebuild_file + + ebuild_file="${SDK_INNER_OVERLAY}/${pkg}/${name}-${version}.ebuild" + enter ebuild "${ebuild_file}" manifest --force +} + +function join_by() { + local delimiter="${1-}" + local first="${2-}" + if shift 2; then + printf '%s' "${first}" "${@/#/${delimiter}}"; + fi +} + +# Generates a changelog entry. Usually the changelog entry is in a +# following form: +# +# - ([]()) +# +# Thus first three parameters of this function should be the name, +# version and URL. The changelog entries are files, so the fourth +# parameter is a name that will be a part of the filename. It often is +# a lower-case variant of the first parameter. +# +# Example: +# generate_update_changelog Go 1.20.2 'https://go.dev/doc/devel/release#go1.20.2' go +# +# Sometimes there's a bigger jump in versions, like from 1.19.1 to +# 1.19.4, so it is possible to pass extra version and URL pairs for +# the intermediate versions: +# +# generate_update_changelog Go 1.19.4 'https://go.dev/doc/devel/release#go1.19.4' go \ +# 1.19.2 'https://go.dev/doc/devel/release#go1.19.2' \ +# 1.19.3 'https://go.dev/doc/devel/release#go1.19.3' +function generate_update_changelog() { + local name="${1}"; shift + local version="${1}"; shift + local url="${1}"; shift + local update_name="${1}"; shift + # rest of parameters are version and link pairs for old versions + local file + local -a old_links + + file="changelog/updates/$(date '+%Y-%m-%d')-${update_name}-${version}-update.md" + + if [[ -d changelog/updates ]]; then + printf '%s %s ([%s](%s)' '-' "${name}" "${version}" "${url}" > "${file}" + if [[ $# -gt 0 ]]; then + echo -n ' (includes ' >> "${file}" + while [[ $# -gt 1 ]]; do + old_links+=( "[${1}](${2})" ) + shift 2 + done + printf '%s' "$(join_by ', ' "${old_links[@]}")" >> "${file}" + echo -n ')' >> "${file}" + fi + echo ')' >> "${file}" + fi +} + +# Regenerates manifest for given package, and commits changes made for +# that package. If there are new entries in changelog directory, these +# are committed too. Another two parameters are old and new versions +# of the package. +# +# Example: +# commit_changes dev-lang/go 1.19.1 1.19.4 +# +# Sometimes more files need to be added to the commit. In such cases +# extra paths can be specified and those will be passed to "git +# add". If an extra path is relative, it will be relative the overlay +# directory in the scripts repo. In order to use globs, it better to +# make sure that that absolute path is passed. +# +# commit_changes dev-lang/go 1.19.1 1.19.4 \ +# some/extra/directory \ +# some/file \ +# "${PWD}/some/globs"*'-suffix' +function commit_changes() { + local pkg="${1}"; shift + local old_version="${1}"; shift + local new_version="${1}"; shift + # rest of parameters are additional directories to add to the commit + local name="${pkg##*/}" + + regenerate_manifest "${pkg}" "${new_version}" + + pushd "${SDK_OUTER_OVERLAY}" + + git add "${pkg}" + if [[ -d changelog ]]; then + git add changelog + fi + for dir; do + git add "${dir}" + done + git commit -m "${pkg}: Update from ${old_version} to ${new_version}" + + popd +} + +# Prints the status of the git repo and cleans it up - reverts +# uncommitted changes, removes untracked files. It's usually called at +# the end of a script making changes to the repository in order to +# avoid unwanted changes to be a part of a PR created by the +# peter-evans/create-pull-request action that follows up. +function cleanup_repo() { + git -C "${SDK_OUTER_OVERLAY}" status + git -C "${SDK_OUTER_OVERLAY}" reset --hard HEAD + git -C "${SDK_OUTER_OVERLAY}" clean -ffdx +} diff --git a/.github/workflows/containerd-apply-patch.sh b/.github/workflows/containerd-apply-patch.sh new file mode 100755 index 0000000000..f7e8f07f52 --- /dev/null +++ b/.github/workflows/containerd-apply-patch.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +pushd "${SDK_OUTER_OVERLAY}" + +VERSION_OLD=$(sed -n "s/^DIST containerd-\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p" app-emulation/containerd/Manifest | sort -ruV | head -n1) +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest Containerd, nothing to do" + exit 0 +fi + +# we need to update not only the main ebuild file, but also its CONTAINERD_COMMIT, +# which needs to point to COMMIT_HASH that matches with $VERSION_NEW from upstream containerd. +containerdEbuildOldSymlink=$(get_ebuild_filename app-emulation/containerd "${VERSION_OLD}") +containerdEbuildNewSymlink="app-emulation/containerd/containerd-${VERSION_NEW}.ebuild" +containerdEbuildMain="app-emulation/containerd/containerd-9999.ebuild" +git mv "${containerdEbuildOldSymlink}" "${containerdEbuildNewSymlink}" +sed -i "s/CONTAINERD_COMMIT=\"\(.*\)\"/CONTAINERD_COMMIT=\"${COMMIT_HASH}\"/g" "${containerdEbuildMain}" +sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${containerdEbuildMain}" + + +DOCKER_VERSION=$(sed -n "s/^DIST docker-\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p" app-emulation/docker/Manifest | sort -ruV | head -n1) +# torcx ebuild file has a docker version with only major and minor versions, like 19.03. +versionTorcx=${DOCKER_VERSION%.*} +torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}") +sed -i "s/containerd-${VERSION_OLD}/containerd-${VERSION_NEW}/g" "${torcxEbuildFile}" + +popd + +URL="https://github.com/containerd/containerd/releases/tag/v${VERSION_NEW}" + +generate_update_changelog 'containerd' "${VERSION_NEW}" "${URL}" 'containerd' + +commit_changes app-emulation/containerd "${VERSION_OLD}" "${VERSION_NEW}" \ + app-torcx/docker + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/containerd-release-main.yaml b/.github/workflows/containerd-release-main.yaml new file mode 100644 index 0000000000..34677ba60c --- /dev/null +++ b/.github/workflows/containerd-release-main.yaml @@ -0,0 +1,49 @@ +name: Get the latest Containerd release for main +on: + schedule: + - cron: '00 8 * * 5' + workflow_dispatch: + +jobs: + get-containerd-release: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + - name: Figure out latest Containerd release version + id: containerd-latest-release + run: | + versionCommitPair=( $(git ls-remote --tags https://github.com/containerd/containerd | grep 'refs/tags/v[0-9]*\.[0-9]*\.[0-9]*$' | sed -e 's#^\([0-9a-fA-F]*\)[[:space:]]*refs/tags/v\(.*\)$#\2 \1#g' | sort --reverse --unique --version-sort | head --lines 1) ) + + echo "VERSION_NEW=${versionCommitPair[0]}" >>"${GITHUB_OUTPUT}" + echo "COMMIT_HASH=${versionCommitPair[1]}" >>"${GITHUB_OUTPUT}" + - name: Set up Flatcar SDK + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + CHANNEL: main + run: scripts/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch for main + id: apply-patch-main + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + VERSION_NEW: ${{ steps.containerd-latest-release.outputs.VERSION_NEW }} + COMMIT_HASH: ${{ steps.containerd-latest-release.outputs.COMMIT_HASH }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: scripts/.github/workflows/containerd-apply-patch.sh + - name: Create pull request for main + uses: peter-evans/create-pull-request@v5 + if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: "containerd-${{ steps.containerd-latest-release.outputs.VERSION_NEW }}-main" + base: main + title: Upgrade Containerd in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.containerd-latest-release.outputs.VERSION_NEW }} + body: Subject says it all. + labels: main diff --git a/.github/workflows/docker-apply-patch.sh b/.github/workflows/docker-apply-patch.sh new file mode 100755 index 0000000000..e5f38d5e09 --- /dev/null +++ b/.github/workflows/docker-apply-patch.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +pushd "${SDK_OUTER_OVERLAY}" + +VERSION_OLD=$(sed -n "s/^DIST docker-\([0-9]*.[0-9]*.[0-9]*\).*/\1/p" app-emulation/docker/Manifest | sort -ruV | head -n1) +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest Docker, nothing to do" + exit 0 +fi + +# we need to update not only the main ebuild file, but also its DOCKER_GITCOMMIT, +# which needs to point to COMMIT_HASH that matches with $VERSION_NEW from upstream docker-ce. +dockerEbuildOld=$(get_ebuild_filename app-emulation/docker "${VERSION_OLD}") +dockerEbuildNew="app-emulation/docker/docker-${VERSION_NEW}.ebuild" +git mv "${dockerEbuildOld}" "${dockerEbuildNew}" +sed -i "s/GIT_COMMIT=\(.*\)/GIT_COMMIT=${COMMIT_HASH_MOBY}/g" "${dockerEbuildNew}" +sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${dockerEbuildNew}" + +cliEbuildOld=$(get_ebuild_filename app-emulation/docker-cli "${VERSION_OLD}") +cliEbuildNew="app-emulation/docker-cli/docker-cli-${VERSION_NEW}.ebuild" +git mv "${cliEbuildOld}" "${cliEbuildNew}" +sed -i "s/GIT_COMMIT=\(.*\)/GIT_COMMIT=${COMMIT_HASH_CLI}/g" "${cliEbuildNew}" +sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${cliEbuildNew}" + +# torcx ebuild file has a docker version with only major and minor versions, like 19.03. +versionTorcx=${VERSION_OLD%.*} +torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}") +sed -i "s/docker-${VERSION_OLD}/docker-${VERSION_NEW}/g" "${torcxEbuildFile}" +sed -i "s/docker-cli-${VERSION_OLD}/docker-cli-${VERSION_NEW}/g" "${torcxEbuildFile}" + +# update also docker versions used by the current docker-runc ebuild file. +versionRunc=$(sed -n "s/^DIST docker-runc-\([0-9]*.[0-9]*.*\)\.tar.*/\1/p" app-emulation/docker-runc/Manifest | sort -ruV | head -n1) +runcEbuildFile=$(get_ebuild_filename app-emulation/docker-runc "${versionRunc}") +sed -i "s/github.com\/docker\/docker-ce\/blob\/v${VERSION_OLD}/github.com\/docker\/docker-ce\/blob\/v${VERSION_NEW}/g" ${runcEbuildFile} + +popd + +# URL for Docker release notes has a specific format of +# https://docs.docker.com/engine/release-notes/MAJOR.MINOR/#COMBINEDFULLVERSION +# To get the subfolder part MAJOR.MINOR, drop the patchlevel of the semver. +# e.g. 20.10.23 -> 20.10 +# To get the combined full version, drop all dots from the full version. +# e.g. 20.10.23 -> 201023 +# So the result becomes like: +# https://docs.docker.com/engine/release-notes/20.10/#201023 +URLSUBFOLDER=${VERSION_NEW%.*} +URLVERSION="${VERSION_NEW//./}" +URL="https://docs.docker.com/engine/release-notes/${URLSUBFOLDER}/#${URLVERSION}" + +generate_update_changelog 'Docker' "${VERSION_NEW}" "${URL}" 'docker' + +regenerate_manifest app-emulation/docker-cli "${VERSION_NEW}" +commit_changes app-emulation/docker "${VERSION_OLD}" "${VERSION_NEW}" \ + app-emulation/docker-cli \ + app-torcx/docker \ + app-emulation/docker-runc + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/docker-release-main.yaml b/.github/workflows/docker-release-main.yaml new file mode 100644 index 0000000000..5459d5a904 --- /dev/null +++ b/.github/workflows/docker-release-main.yaml @@ -0,0 +1,52 @@ +name: Get the latest Docker release for main +on: + schedule: + - cron: '35 7 * * 3' + workflow_dispatch: + +jobs: + get-docker-release: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + - name: Figure out latest Docker release version + id: docker-latest-release + run: | + versionCommitPairMoby=( $(git ls-remote --tags https://github.com/moby/moby | grep 'refs/tags/v[0-9]*\.[0-9]*\.[0-9]*$' | sed -e 's#^\([0-9a-fA-F]*\)[[:space:]]*refs/tags/v\(.*\)$#\2 \1#g' | sort --reverse --unique --version-sort | head --lines 1) ) + commitHashCLI=$(git ls-remote --tags https://github.com/docker/cli | grep 'refs/tags/v'"${versionCommitPairMoby[0]}"'$' | cut -f1) + + echo "VERSION_NEW=${versionCommitPairMoby[0]}" >>"${GITHUB_OUTPUT}" + echo "COMMIT_HASH_MOBY=${versionCommitPairMoby[1]}" >>"${GITHUB_OUTPUT}" + echo "COMMIT_HASH_CLI=${commitHashCLI}" >>"${GITHUB_OUTPUT}" + - name: Set up Flatcar SDK + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + CHANNEL: main + run: scripts/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch for main + id: apply-patch-main + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + VERSION_NEW: ${{ steps.docker-latest-release.outputs.VERSION_NEW }} + COMMIT_HASH_MOBY: ${{ steps.docker-latest-release.outputs.COMMIT_HASH_MOBY }} + COMMIT_HASH_CLI: ${{ steps.docker-latest-release.outputs.COMMIT_HASH_CLI }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: scripts/.github/workflows/docker-apply-patch.sh + - name: Create pull request for main + uses: peter-evans/create-pull-request@v5 + if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: docker-${{ steps.docker-latest-release.outputs.VERSION_NEW }}-main + base: main + title: Upgrade Docker in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.docker-latest-release.outputs.VERSION_NEW }} + body: Subject says it all. + labels: main diff --git a/.github/workflows/figure-out-branch.sh b/.github/workflows/figure-out-branch.sh new file mode 100755 index 0000000000..abf7365ab5 --- /dev/null +++ b/.github/workflows/figure-out-branch.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Prints the following github outputs based on channel named passed to +# the script as a parameter. +# +# BRANCH is a name of the git branch related to the passed channel. +# +# SKIP tells whether the rest of the steps should be skipped, will be +# either 0 or 1. +# +# LINK is a link to release mirror for the following channel. Will be +# empty for main channel. +# +# LABEL is going to be mostly the same as the channel name, except +# that lts-old will be labeled as lts. + +set -euo pipefail + +if [[ ${#} -ne 1 ]]; then + echo "Expected a channel name as a parameter" >&2 + exit 1 +fi + +channel_name="${1}" +skip=0 +link='' +branch='' +label='' +case "${channel_name}" in + main) + branch='main' + ;; + lts-old) + curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 'https://lts.release.flatcar-linux.net/lts-info' + if [[ $(grep -e ':supported' lts-info | wc -l) -le 1 ]]; then + # Only one supported LTS, skip this workflow run + # as 'lts' matrix branch will handle updating the only + # supported LTS. + skip=1 + else + line=$(grep -e ':supported' lts-info | sort -V | head -n 1) + major=$(awk -F: '{print $1}' <<<"${line}") + year=$(awk -F: '{print $2}' <<<"${line}") + branch="flatcar-${major}" + link="https://lts.release.flatcar-linux.net/amd64-usr/current-${year}" + label='lts' + fi + rm -f lts-info + ;; + alpha|beta|stable|lts) + link="https://${channel_name}.release.flatcar-linux.net/amd64-usr/current" + major=$(curl -sSL "${link}/version.txt" | awk -F= '/FLATCAR_BUILD=/{ print $2 }') + branch="flatcar-${major}" + ;; + *) + echo "Unknown channel '${channel_name}'" >&2 + exit 1 +esac + +if [[ -z "${label}" ]]; then + label="${channel_name}" +fi + +echo "BRANCH=${branch}" >>"${GITHUB_OUTPUT}" +echo "SKIP=${skip}" >>"${GITHUB_OUTPUT}" +echo "LINK=${link}" >>"${GITHUB_OUTPUT}" +echo "LABEL=${label}" >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/firmware-apply-patch.sh b/.github/workflows/firmware-apply-patch.sh new file mode 100755 index 0000000000..424d85a941 --- /dev/null +++ b/.github/workflows/firmware-apply-patch.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +pushd "${SDK_OUTER_OVERLAY}" + +# Parse the Manifest file for already present source files and keep the latest version in the current series +VERSION_OLD=$(sed -n "s/^DIST linux-firmware-\([0-9]*\).*$/\1/p" sys-kernel/coreos-firmware/Manifest | sort -ruV | head -n1) +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest Linux Firmware, nothing to do" + exit 0 +fi + +EBUILD_FILENAME=$(get_ebuild_filename sys-kernel/coreos-firmware "${VERSION_OLD}") +git mv "${EBUILD_FILENAME}" "sys-kernel/coreos-firmware/coreos-firmware-${VERSION_NEW}.ebuild" + +popd + +URL="https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tag/?h=${VERSION_NEW}" + +generate_update_changelog 'Linux Firmware' "${VERSION_NEW}" "${URL}" 'linux-firmware' + +commit_changes sys-kernel/coreos-firmware "${VERSION_OLD}" "${VERSION_NEW}" + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/firmware-release-main.yaml b/.github/workflows/firmware-release-main.yaml new file mode 100644 index 0000000000..4d90671c81 --- /dev/null +++ b/.github/workflows/firmware-release-main.yaml @@ -0,0 +1,46 @@ +name: Get the latest Linux Firmware release for main +on: + schedule: + - cron: '0 7 * * 4' + workflow_dispatch: + +jobs: + get-firmware-release: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + - name: Figure out latest Linux Firmware release version + id: firmware-latest-release + run: | + versionNew=$(git ls-remote --tags https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git | cut -f2 | sed -n "/refs\/tags\/[0-9]*$/s/^refs\/tags\///p" | sort -ruV | head -n1) + echo "VERSION_NEW=${versionNew}" >>"${GITHUB_OUTPUT}" + - name: Set up Flatcar SDK + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + CHANNEL: main + run: scripts/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch for main + id: apply-patch-main + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + VERSION_NEW: ${{ steps.firmware-latest-release.outputs.VERSION_NEW }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: scripts/.github/workflows/firmware-apply-patch.sh + - name: Create pull request for main + uses: peter-evans/create-pull-request@v5 + if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: firmware-${{ steps.firmware-latest-release.outputs.VERSION_NEW }}-main + base: main + title: Upgrade Linux Firmware in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.firmware-latest-release.outputs.VERSION_NEW }} + body: Subject says it all. + labels: main diff --git a/.github/workflows/go-apply-patch.sh b/.github/workflows/go-apply-patch.sh new file mode 100755 index 0000000000..f22dfd1b1f --- /dev/null +++ b/.github/workflows/go-apply-patch.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +# create a mapping between short version and new version, e.g. 1.16 -> 1.16.3 +declare -A VERSIONS +for version_new in ${VERSIONS_NEW}; do + version_new_trimmed="${version_new%.*}" + if [[ "${version_new_trimmed%.*}" = "${version_new_trimmed}" ]]; then + version_new_trimmed="${version_new}" + fi + VERSIONS["${version_new_trimmed}"]="${version_new}" +done + +branch_name="go-$(join_by '-and-' ${VERSIONS_NEW})-main" + +# Parse the Manifest file for already present source files and keep the latest version in the current series +# DIST go1.17.src.tar.gz ... => 1.17 +# DIST go1.17.1.src.tar.gz ... => 1.17.1 +declare -a UPDATED_VERSIONS_OLD UPDATED_VERSIONS_NEW +any_different=0 +for version_short in "${!VERSIONS[@]}"; do + pushd "${SDK_OUTER_OVERLAY}" + VERSION_NEW="${VERSIONS["${version_short}"]}" + VERSION_OLD=$(sed -n "s/^DIST go\(${version_short}\(\.*[0-9]*\)\?\)\.src.*/\1/p" dev-lang/go/Manifest | sort -ruV | head -n1) + if [[ -z "${VERSION_OLD}" ]]; then + echo "${version_short} is not packaged, skipping" + popd + continue + fi + if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "${version_short} is already at the latest (${VERSION_NEW}), skipping" + popd + continue + fi + UPDATED_VERSIONS_OLD+=("${VERSION_OLD}") + UPDATED_VERSIONS_NEW+=("${VERSION_NEW}") + + any_different=1 + EBUILD_FILENAME=$(get_ebuild_filename dev-lang/go "${VERSION_OLD}") + git mv "${EBUILD_FILENAME}" "dev-lang/go/go-${VERSION_NEW}.ebuild" + + popd + + URL="https://go.dev/doc/devel/release#go${VERSION_NEW}" + + generate_update_changelog 'Go' "${VERSION_NEW}" "${URL}" 'go' + + commit_changes dev-lang/go "${VERSION_OLD}" "${VERSION_NEW}" +done + +cleanup_repo + +if [[ $any_different -eq 0 ]]; then + echo "go packages were already at the latest versions, nothing to do" + exit 0 +fi + +vo_gh="$(join_by ' and ' "${UPDATED_VERSIONS_OLD[@]}")" +vn_gh="$(join_by ' and ' "${UPDATED_VERSIONS_NEW[@]}")" + +echo "VERSIONS_OLD=${vo_gh}" >>"${GITHUB_OUTPUT}" +echo "VERSIONS_NEW=${vn_gh}" >>"${GITHUB_OUTPUT}" +echo "BRANCH_NAME=${branch_name}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/go-current-major-versions.sh b/.github/workflows/go-current-major-versions.sh new file mode 100755 index 0000000000..d279161b76 --- /dev/null +++ b/.github/workflows/go-current-major-versions.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +pushd "${SDK_OUTER_OVERLAY}" + +versions=() +for ebuild in dev-lang/go/go-*.ebuild; do + version="${ebuild##*/go-}" # 1.20.1-r1.ebuild or 1.19.ebuild + version="${version%.ebuild}" # 1.20.1-r1 or 1.19 + version="${version%%-*}" # 1.20.1 or 1.19 + short_version="${version%.*}" # 1.20 or 1 + if [[ "${short_version%.*}" = "${short_version}" ]]; then + # fix short version + short_version="${version}" + fi + + versions+=($(git ls-remote --tags https://github.com/golang/go | \ + cut -f2 | \ + sed --quiet "/refs\/tags\/go${short_version}\(\.[0-9]*\)\?$/s/^refs\/tags\/go//p" | \ + grep --extended-regexp --invert-match --regexp='(beta|rc)' | \ + sort --reverse --unique --version-sort | \ + head --lines=1)) +done + +popd + +echo "VERSIONS_NEW=${versions[*]}" >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/go-release-main.yaml b/.github/workflows/go-release-main.yaml new file mode 100644 index 0000000000..e2e43310e1 --- /dev/null +++ b/.github/workflows/go-release-main.yaml @@ -0,0 +1,47 @@ +name: Get the latest Go release for main +on: + schedule: + - cron: '15 7 * * 1' + workflow_dispatch: + +jobs: + get-go-releases: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + - name: Figure out latest Go release versions + id: go-latest-release + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + run: scripts/.github/workflows/go-current-major-versions.sh + - name: Set up Flatcar SDK + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + CHANNEL: main + run: scripts/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch for main + id: apply-patch-main + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + VERSIONS_NEW: ${{ steps.go-latest-release.outputs.VERSIONS_NEW }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: scripts/.github/workflows/go-apply-patch.sh + - name: Create pull request for main + uses: peter-evans/create-pull-request@v5 + if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: ${{ steps.apply-patch-main.outputs.BRANCH_NAME }} + base: main + title: Upgrade Go from ${{ steps.apply-patch-main.outputs.VERSIONS_OLD }} to ${{ steps.apply-patch-main.outputs.VERSIONS_NEW }} + body: Subject says it all. + labels: main diff --git a/.github/workflows/kernel-apply-patch.sh b/.github/workflows/kernel-apply-patch.sh new file mode 100755 index 0000000000..282f0fffdc --- /dev/null +++ b/.github/workflows/kernel-apply-patch.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +pushd "${SDK_OUTER_OVERLAY}" + +# trim the 3rd part in the input semver, e.g. from 5.4.1 to 5.4 +VERSION_SHORT=${VERSION_NEW%.*} +VERSION_OLD=$(sed -n "s/^DIST patch-\(${VERSION_SHORT}\.[0-9]*\).*/\1/p" sys-kernel/coreos-sources/Manifest) +if [[ -z "${VERSION_OLD}" ]]; then + VERSION_OLD=$(sed -n "s/^DIST linux-\(${VERSION_SHORT}*\).*/\1/p" sys-kernel/coreos-sources/Manifest) +fi +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest Kernel, nothing to do" + exit 0 +fi + +for pkg in sources modules kernel; do + pushd "sys-kernel/coreos-${pkg}" + git mv "coreos-${pkg}"-*.ebuild "coreos-${pkg}-${VERSION_NEW}.ebuild" + sed -i -e '/^COREOS_SOURCE_REVISION=/s/=.*/=""/' "coreos-${pkg}-${VERSION_NEW}.ebuild" + popd +done + +popd + +function get_lwn_link() { + local LINUX_VERSION="${1}"; shift + local url + + if ! curl -sfA 'Chrome' -L 'http://www.google.com/search?hl=en&q=site%3Alwn.net+linux+'"${LINUX_VERSION}" -o search.html >&2; then + echo 'curl failed' >&2 + touch search.html + fi + # can't use grep -m 1 -o … to replace head -n 1, because all the links + # seem to happen in one line, so grep prints all the links in the line + url=$({ grep -o 'https://lwn.net/Articles/[0-9]\+' search.html || true ; } | head -n 1) + if [[ ! "${url}" ]]; then + echo 'no valid links found in the search result' >&2 + url="https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tag/?h=v${LINUX_VERSION}" + fi + rm search.html + echo "${url}" +} + +PATCH_VERSION_OLD=${VERSION_OLD##*.} +PATCH_VERSION_NEW=${VERSION_NEW##*.} + +PATCH_NUM=$((PATCH_VERSION_NEW - 1)) + +OLD_VERSIONS_AND_URLS=() + +while [[ ${PATCH_NUM} -gt ${PATCH_VERSION_OLD} ]]; do + TMP_VERSION="${VERSION_SHORT}.${PATCH_NUM}" + TMP_URL=$(get_lwn_link "${TMP_VERSION}") + OLD_VERSIONS_AND_URLS+=( "${TMP_VERSION}" "${TMP_URL}" ) + : $((PATCH_NUM--)) +done + +URL=$(get_lwn_link "${VERSION_NEW}") + +generate_update_changelog 'Linux' "${VERSION_NEW}" "${URL}" 'linux' "${OLD_VERSIONS_AND_URLS[@]}" + +commit_changes sys-kernel/coreos-sources "${VERSION_OLD}" "${VERSION_NEW}" \ + sys-kernel/coreos-modules \ + sys-kernel/coreos-kernel + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/kernel-current-major-version.sh b/.github/workflows/kernel-current-major-version.sh new file mode 100755 index 0000000000..a159b16f73 --- /dev/null +++ b/.github/workflows/kernel-current-major-version.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +pushd "${SDK_OUTER_OVERLAY}" + +KV=$(git ls-files 'sys-kernel/coreos-kernel/*ebuild' | head -n 1 | cut -d '-' -f 5- | cut -d . -f 1-2) +REMOTE='https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git' +kernelVersion=$(git ls-remote --tags "${REMOTE}" | cut -f2 | sed -n "/refs\/tags\/v${KV}\.[0-9]*$/s/^refs\/tags\/v//p" | sort -ruV | head -1) + +popd + +echo "KERNEL_VERSION=${kernelVersion}" >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/kernel-release.yaml b/.github/workflows/kernel-release.yaml new file mode 100644 index 0000000000..1c1f973c80 --- /dev/null +++ b/.github/workflows/kernel-release.yaml @@ -0,0 +1,68 @@ +name: Get the latest Kernel release for all maintained branches +on: + schedule: + - cron: '0 7 * * *' + workflow_dispatch: + +jobs: + get-kernel-release: + strategy: + matrix: + channel: [main,alpha,beta,stable,lts,lts-old] + fail-fast: false + runs-on: ubuntu-latest + steps: + - name: Check out main scripts branch for GitHub workflow scripts only + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: gha + ref: main + - name: Figure out branch + id: figure-out-branch + run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}' + - name: Check out work scripts branch for updating + if: steps.figure-out-branch.outputs.SKIP == 0 + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: work + ref: ${{ steps.figure-out-branch.outputs.BRANCH }} + - name: Figure out latest Linux release version + if: steps.figure-out-branch.outputs.SKIP == 0 + id: kernel-latest-release + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/gha" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/work" + run: gha/.github/workflows/kernel-current-major-version.sh + - name: Set up Flatcar SDK + if: steps.figure-out-branch.outputs.SKIP == 0 + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/work" + CHANNEL: ${{ steps.figure-out-branch.outputs.LABEL }} + # This will be empty for the main channel, but we handle + # this case inside setup-flatcar-sdk.sh. + MIRROR_LINK: ${{ steps.figure-out-branch.outputs.LINK }} + run: gha/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch + if: steps.figure-out-branch.outputs.SKIP == 0 + id: apply-patch + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/gha" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/work" + VERSION_NEW: ${{ steps.kernel-latest-release.outputs.KERNEL_VERSION }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: gha/.github/workflows/kernel-apply-patch.sh + - name: Create pull request + if: (steps.figure-out-branch.outputs.SKIP == 0) && (steps.apply-patch.outputs.UPDATE_NEEDED == 1) + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: work + branch: "linux-${{ steps.kernel-latest-release.outputs.KERNEL_VERSION }}-${{ steps.figure-out-branch.outputs.BRANCH }}" + base: ${{ steps.figure-out-branch.outputs.BRANCH }} + title: Upgrade Linux Kernel for ${{ steps.figure-out-branch.outputs.BRANCH }} from ${{ steps.apply-patch.outputs.VERSION_OLD }} to ${{ steps.kernel-latest-release.outputs.KERNEL_VERSION }} + body: Subject says it all. + labels: ${{ steps.figure-out-branch.outputs.LABEL }} diff --git a/.github/workflows/mirror-calico.sh b/.github/workflows/mirror-calico.sh new file mode 100644 index 0000000000..0218af1559 --- /dev/null +++ b/.github/workflows/mirror-calico.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# This script will mirror the list of Calico images +# from Docker Hub to GHCR. + +# tag will hold the version of calico images we +# previously fetched +tag="${1}" + +# list of images to mirror from Docker Hub +images=( + calico/typha + calico/pod2daemon-flexvol + calico/cni + calico/node + calico/kube-controllers +) + +# we iterate over the images we want to mirror +for image in "${images[@]}"; do + ./mirror-to-ghcr.sh "${image}" "${tag}" +done diff --git a/.github/workflows/mirror-calico.yaml b/.github/workflows/mirror-calico.yaml new file mode 100644 index 0000000000..564871b040 --- /dev/null +++ b/.github/workflows/mirror-calico.yaml @@ -0,0 +1,40 @@ +name: Sync GHCR Calico images with Docker Hub +on: + schedule: + # run every 12h + - cron: '0 */12 * * *' + workflow_dispatch: + +jobs: + mirror-calico: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + - name: Login to GitHub Container Registry (ghcr) + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_PASSWORD }} + - name: Figure out latest Calico release version + id: calico-latest-release + run: | + set -exuo pipefail + + calico_version=$(curl \ + -H 'Accept: application/vnd.github+json' \ + 'https://api.github.com/repos/projectcalico/calico/releases' | \ + jq --raw-output '.[].tag_name' | \ + sort --version-sort --reverse | \ + head --lines=1) + + echo "Found version: ${calico_version}" + echo "CALICO_VERSION=${calico_verison}" >>"${GITHUB_OUTPUT}" + - name: Mirror calico images to GHCR + env: + CALICO_VERSION: ${{ steps.calico-latest-release.outputs.CALICO_VERSION }} + run: | + pushd .github/workflows/ + ./mirror-calico.sh "${CALICO_VERSION}" + popd diff --git a/.github/workflows/mirror-to-ghcr.sh b/.github/workflows/mirror-to-ghcr.sh new file mode 100644 index 0000000000..2b413af021 --- /dev/null +++ b/.github/workflows/mirror-to-ghcr.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# This generic script aims to mirror an image from Docker hub to another registry. +# Authentication to the registry must be done before. + +image="${1}" +imagetag="${2}" +org="${3:-kinvolk}" + +# we want both arch for running tests +platforms=( amd64 arm64 ) + +# tags will hold the mirrored images +tags=() + +name="ghcr.io/${org}/${image}:${imagetag}" + +for platform in "${platforms[@]}"; do + # we first fetch the image from Docker Hub + var=$(docker pull "${image}:${imagetag}" --platform="linux/${platform}" -q) + # we prepare the image to be pushed into another registry + tag="${name}-${platform}" + # we tag the image to create the mirrored image + docker tag "${var}" "${tag}" + docker push "${tag}" + tags+=( "${tag}" ) +done + +docker manifest create "${name}" "${tags[@]}" +# some images have bad arch specs in the individual image manifests :( +docker manifest annotate "${name}" "${name}-arm64" --arch arm64 +docker manifest push --purge "${name}" diff --git a/.github/workflows/portage-stable-packages-list b/.github/workflows/portage-stable-packages-list new file mode 100644 index 0000000000..53376ea92a --- /dev/null +++ b/.github/workflows/portage-stable-packages-list @@ -0,0 +1,383 @@ +# Please keep the list sorted! + +acct-group/adm +acct-group/audio +acct-group/cdrom +acct-group/dialout +acct-group/disk +acct-group/dnsmasq +acct-group/input +acct-group/kmem +acct-group/kvm +acct-group/lp +acct-group/man +acct-group/messagebus +acct-group/netperf +acct-group/nobody +acct-group/ntp +acct-group/pcap +acct-group/polkitd +acct-group/portage +acct-group/render +acct-group/root +acct-group/sgx +acct-group/sshd +acct-group/systemd-coredump +acct-group/systemd-journal +acct-group/systemd-journal-remote +acct-group/systemd-network +acct-group/systemd-oom +acct-group/systemd-resolve +acct-group/systemd-timesync +acct-group/tape +acct-group/tss +acct-group/tty +acct-group/users +acct-group/utmp +acct-group/video +acct-group/wheel + +acct-user/dnsmasq +acct-user/man +acct-user/messagebus +acct-user/netperf +acct-user/nobody +acct-user/ntp +acct-user/pcap +acct-user/polkitd +acct-user/portage +acct-user/root +acct-user/sshd +acct-user/systemd-coredump +acct-user/systemd-journal-remote +acct-user/systemd-network +acct-user/systemd-oom +acct-user/systemd-resolve +acct-user/systemd-timesync +acct-user/tss + +app-alternatives/awk +app-alternatives/bc +app-alternatives/bzip2 +app-alternatives/cpio +app-alternatives/gzip +app-alternatives/lex +app-alternatives/sh +app-alternatives/tar +app-alternatives/yacc + +app-arch/bzip2 +app-arch/cpio +app-arch/gzip +app-arch/lbzip2 +app-arch/libarchive +app-arch/ncompress +app-arch/pbzip2 +app-arch/pigz +app-arch/rpm2targz +app-arch/sharutils +app-arch/tar +app-arch/unzip +app-arch/xz-utils +app-arch/zip +app-arch/zstd + +app-cdr/cdrtools + +app-crypt/adcli +app-crypt/mit-krb5 + +app-editors/vim +app-editors/vim-core + +app-emulation/qemu +app-emulation/qemu-guest-agent + +app-eselect/eselect-iptables + +app-portage/portage-utils + +app-shells/bash-completion + +app-text/asciidoc +app-text/build-docbook-catalog +app-text/docbook-xml-dtd +app-text/docbook-xsl-ns-stylesheets +app-text/docbook-xsl-stylesheets +app-text/sgml-common + +dev-db/sqlite + +dev-lang/duktape +dev-lang/perl +dev-lang/python +dev-lang/python-exec +dev-lang/python-exec-conf + +dev-libs/boost +dev-libs/cJSON +dev-libs/cyrus-sasl +dev-libs/elfutils +dev-libs/expat +dev-libs/glib +dev-libs/gmp +dev-libs/gobject-introspection +dev-libs/gobject-introspection-common +dev-libs/jsoncpp +dev-libs/libaio +dev-libs/libksba +dev-libs/libltdl +dev-libs/libnl +dev-libs/libpcre2 +dev-libs/libtasn1 +dev-libs/libuv +dev-libs/libxml2 +dev-libs/libxslt +dev-libs/nettle +dev-libs/oniguruma + +dev-perl/File-Slurp +dev-perl/Locale-gettext +dev-perl/Parse-Yapp +dev-perl/Text-Unidecode +dev-perl/Unicode-EastAsianWidth + +dev-python/autocommand +dev-python/boto +dev-python/certifi +dev-python/crcmod +dev-python/cython +dev-python/distro +dev-python/docutils +dev-python/fasteners +dev-python/flit_core +dev-python/gentoo-common +dev-python/gpep517 +dev-python/inflect +dev-python/installer +dev-python/jaraco-context +dev-python/jaraco-functools +dev-python/jaraco-text +dev-python/jinja +dev-python/lazy-object-proxy +dev-python/lxml +dev-python/markupsafe +dev-python/more-itertools +dev-python/nspektr +dev-python/ordered-set +dev-python/packaging +dev-python/platformdirs +dev-python/pydantic +dev-python/pydecomp +dev-python/pygments +dev-python/pyparsing +dev-python/setuptools +dev-python/setuptools-scm +dev-python/six +dev-python/snakeoil +dev-python/tomli +dev-python/typing-extensions +dev-python/wheel + +dev-util/b2 +dev-util/bpftool +dev-util/catalyst +dev-util/checkbashisms +dev-util/cmake +dev-util/cmocka +dev-util/desktop-file-utils +dev-util/gdbus-codegen +dev-util/glib-utils +dev-util/gperf +dev-util/gtk-doc-am +dev-util/meson +dev-util/meson-format-array +dev-util/ninja +dev-util/pahole +dev-util/patchelf +dev-util/patchutils +dev-util/perf +dev-util/pkgconf +dev-util/re2c +dev-util/strace + +dev-vcs/git +dev-vcs/repo + +eclass/acct-group.eclass +eclass/acct-user.eclass +eclass/alternatives.eclass +eclass/app-alternatives.eclass +eclass/autotools.eclass +# Still has some Flatcar modifications, will need to upstream it first. +# +# eclass/bash-completion-r1.eclass +eclass/cmake-multilib.eclass +eclass/cmake.eclass +eclass/desktop.eclass +eclass/distutils-r1.eclass +eclass/eapi7-ver.eclass +eclass/eapi8-dosym.eclass +eclass/edo.eclass +eclass/edos2unix.eclass +eclass/elisp-common.eclass +eclass/epatch.eclass +eclass/eqawarn.eclass +eclass/estack.eclass +eclass/eutils.eclass +eclass/fcaps.eclass +eclass/flag-o-matic.eclass +eclass/git-r3.eclass +eclass/gnome.org.eclass +eclass/gnuconfig.eclass +eclass/java-pkg-opt-2.eclass +eclass/java-utils-2.eclass +eclass/kernel-2.eclass +eclass/libtool.eclass +eclass/linux-info.eclass +eclass/linux-mod.eclass +eclass/llvm.eclass +eclass/ltprune.eclass +eclass/meson-multilib.eclass +eclass/meson.eclass +eclass/multibuild.eclass +eclass/multilib-build.eclass +eclass/multilib-minimal.eclass +eclass/multilib.eclass +eclass/multiprocessing.eclass +eclass/ninja-utils.eclass +eclass/optfeature.eclass +eclass/out-of-source-utils.eclass +eclass/pam.eclass +eclass/pax-utils.eclass +eclass/perl-functions.eclass +eclass/portability.eclass +eclass/plocale.eclass +eclass/prefix.eclass +eclass/preserve-libs.eclass +eclass/pypi.eclass +eclass/python-any-r1.eclass +eclass/python-r1.eclass +eclass/python-single-r1.eclass +eclass/python-utils-r1.eclass +eclass/readme.gentoo-r1.eclass +eclass/savedconfig.eclass +eclass/strip-linguas.eclass +eclass/systemd.eclass +eclass/tmpfiles.eclass +eclass/toolchain-funcs.eclass +eclass/toolchain.eclass +eclass/udev.eclass +eclass/user-info.eclass +# This file is modified by us to be an empty file, so can't be synced for now. +# +# eclass/usr-ldscript.eclass +eclass/vcs-clean.eclass +eclass/verify-sig.eclass +eclass/vim-doc.eclass +eclass/vim-plugin.eclass +eclass/virtualx.eclass +eclass/waf-utils.eclass +eclass/wrapper.eclass +eclass/xdg-utils.eclass + +licenses + +media-libs/libpng + +net-analyzer/nmap +net-analyzer/traceroute + +net-dns/bind-tools +net-dns/dnsmasq + +net-fs/cifs-utils + +net-libs/gnutls +net-libs/libpcap +net-libs/libslirp +net-libs/nghttp2 + +net-misc/bridge-utils +net-misc/curl +net-misc/ethertypes +net-misc/iperf +net-misc/iputils +net-misc/rsync +net-misc/socat +net-misc/wget +net-misc/whois + +net-vpn/wireguard-tools + +perl-core/File-Temp + +profiles + +# The bootstrap script has some modifications, so we can't sync scripts directory yet. +# +# scripts + +sys-apps/attr +sys-apps/diffutils +sys-apps/dtc +sys-apps/file +sys-apps/findutils +sys-apps/gawk +sys-apps/gentoo-functions +sys-apps/help2man +sys-apps/i2c-tools +sys-apps/iucode_tool +sys-apps/less +sys-apps/portage +sys-apps/texinfo + +sys-devel/autoconf +sys-devel/autoconf-archive +sys-devel/autoconf-wrapper +sys-devel/automake +sys-devel/automake-wrapper +sys-devel/bc +sys-devel/binutils +sys-devel/binutils-config +sys-devel/bison +sys-devel/crossdev +sys-devel/flex +sys-devel/gcc +sys-devel/gcc-config +sys-devel/gettext +sys-devel/gnuconfig +sys-devel/libtool +sys-devel/m4 +sys-devel/patch + +sys-firmware/edk2-ovmf-bin +sys-firmware/intel-microcode +sys-firmware/ipxe +sys-firmware/seabios-bin +sys-firmware/sgabios + +sys-kernel/linux-headers + +sys-fs/e2fsprogs +sys-fs/multipath-tools + +sys-libs/binutils-libs +sys-libs/libcap +sys-libs/libcap-ng +sys-libs/libseccomp +sys-libs/zlib + +virtual/libcrypt +virtual/libelf +virtual/perl-Carp +virtual/perl-Exporter +virtual/perl-ExtUtils-MakeMaker +virtual/perl-File-Spec +virtual/perl-File-Temp +virtual/perl-Getopt-Long +virtual/perl-IO +virtual/perl-version +virtual/pkgconfig + +x11-libs/pixman diff --git a/.github/workflows/runc-apply-patch.sh b/.github/workflows/runc-apply-patch.sh new file mode 100755 index 0000000000..f89ef5cf57 --- /dev/null +++ b/.github/workflows/runc-apply-patch.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +pushd "${SDK_OUTER_OVERLAY}" + +# Get the newest runc version, including official releases and rc +# versions. We need some sed tweaks like replacing dots with +# underscores, adding trailing underscore, sort, and trim the trailing +# underscore and replace other underscores with dots again, so that +# sort -V can properly sort "1.0.0" as newer than "1.0.0-rc95" and +# "0.0.2.1" as newer than "0.0.2". +VERSION_OLD=$(sed -n "s/^DIST docker-runc-\([0-9]*\.[0-9]*.*\)\.tar.*/\1_/p" app-emulation/docker-runc/Manifest | tr '.' '_' | sort -ruV | sed -e 's/_$//' | tr '_' '.' | head -n1) +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest Runc, nothing to do" + exit 0 +fi + +runcEbuildOld=$(get_ebuild_filename app-emulation/docker-runc "${VERSION_OLD}") +runcEbuildNew="app-emulation/docker-runc/docker-runc-${VERSION_NEW}.ebuild" +git mv "${runcEbuildOld}" "${runcEbuildNew}" +sed -i "s/${VERSION_OLD}/${VERSION_NEW}/g" "${runcEbuildNew}" +sed -i "s/COMMIT_ID=\"\(.*\)\"/COMMIT_ID=\"${COMMIT_HASH}\"/g" "${runcEbuildNew}" + +# update also runc versions used by docker and containerd +sed -i "s/docker-runc-${VERSION_OLD}/docker-runc-${VERSION_NEW}/g" app-emulation/containerd/containerd-9999.ebuild + +dockerVersion=$(sed -n "s/^DIST docker-\([0-9]*.[0-9]*.[0-9]*\).*/\1/p" app-emulation/docker/Manifest | sort -ruV | head -n1) + +# torcx ebuild file has a docker version with only major and minor versions, like 19.03. +versionTorcx=${dockerVersion%.*} +torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}") +sed -i "s/docker-runc-${VERSION_OLD}/docker-runc-${VERSION_NEW}/g" "${torcxEbuildFile}" + +popd + +URL="https://github.com/opencontainers/runc/releases/tag/v${VERSION_NEW}" + +generate_update_changelog 'runc' "${VERSION_NEW}" "${URL}" 'runc' + +commit_changes app-emulation/docker-runc "${VERSION_OLD}" "${VERSION_NEW}" \ + app-emulation/containerd \ + app-torcx/docker + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/runc-release-main.yaml b/.github/workflows/runc-release-main.yaml new file mode 100644 index 0000000000..db19a56e81 --- /dev/null +++ b/.github/workflows/runc-release-main.yaml @@ -0,0 +1,64 @@ +name: Get the latest Runc release for main +on: + schedule: + - cron: '50 7 * * 4' + workflow_dispatch: + +jobs: + get-runc-release: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + - name: Figure out latest Runc release version + id: runc-latest-release + run: | + REMOTE='https://github.com/opencontainers/runc' + # Get the newest runc version, including official releases + # and rc versions. We need some sed tweaks like replacing + # dots with underscores, adding trailing underscore, sort, + # and trim the trailing underscore and replace other + # underscores with dots again, so that sort -V can properly + # sort "1.0.0" as newer than "1.0.0-rc95" and "0.0.2.1" as + # newer than "0.0.2". + versionCommitPair=( $(git ls-remote --tags "${REMOTE}" | grep 'refs/tags/v[a-z0-9._-]*$' | sed -e 's#^\([0-9a-fA-F]*\)[[:space:]]*refs/tags/v\(.*\)$#\2_ \1#g' -e 's/\./_/g' | sort --reverse --unique --version-sort --key=1,1 | sed -e 's/_ / /' -e 's/_/./g' | head --lines=1) ) + versionNew="${versionCommitPair[0]}" + # Gentoo expects an underline between version and rc, so + # "1.1.0-rc.1" becomes "1.1.0_rc.1". + versionNew="${versionNew//-/_}" + # Gentoo expects no separators between rc and the number, so + # "1.1.0_rc.1" becomes "1.1.0_rc1" + versionNew="${versionNew//rc./rc}" + commitHash="${versionCommitPair[1]}" + echo "VERSION_NEW=${versionNew}" >>"${GITHUB_OUTPUT}" + echo "COMMIT_HASH=${commitHash}" >>"${GITHUB_OUTPUT}" + - name: Set up Flatcar SDK + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + CHANNEL: main + run: scripts/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch for main + id: apply-patch-main + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + VERSION_NEW: ${{ steps.runc-latest-release.outputs.VERSION_NEW }} + COMMIT_HASH: ${{ steps.runc-latest-release.outputs.COMMIT_HASH }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: scripts/.github/workflows/runc-apply-patch.sh + - name: Create pull request for main + uses: peter-evans/create-pull-request@v5 + if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: runc-${{ steps.runc-latest-release.outputs.VERSION_NEW }}-main + base: main + title: Upgrade Runc in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.runc-latest-release.outputs.VERSION_NEW }} + body: Subject says it all. + labels: main diff --git a/.github/workflows/rust-apply-patch.sh b/.github/workflows/rust-apply-patch.sh new file mode 100755 index 0000000000..1b51ce6335 --- /dev/null +++ b/.github/workflows/rust-apply-patch.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +pushd "${SDK_OUTER_OVERLAY}" + +VERSION_OLD=$(sed -n "s/^DIST rustc-\(1\.[0-9]*\.[0-9]*\).*/\1/p" dev-lang/rust/Manifest | sort -ruV | head -n1) +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest Rust, nothing to do" + exit 0 +fi + +# Replace (dev-lang/virtual)/rust versions in profiles/, e.g. package.accept_keywords. +# Try to match all kinds of version specifiers, e.g. >=, <=, =, ~. +find profiles -name 'package.*' | xargs sed -i "s/\([><]*=\|~\)*dev-lang\/rust-\S\+/\1dev-lang\/rust-${VERSION_NEW}/" +find profiles -name 'package.*' | xargs sed -i "s/\([><]*=\|~\)*virtual\/rust-\S\+/\1virtual\/rust-${VERSION_NEW}/" + +EBUILD_FILENAME=$(get_ebuild_filename dev-lang/rust "${VERSION_OLD}") +git mv "${EBUILD_FILENAME}" "dev-lang/rust/rust-${VERSION_NEW}.ebuild" +EBUILD_FILENAME=$(get_ebuild_filename virtual/rust "${VERSION_OLD}") +git mv "${EBUILD_FILENAME}" "virtual/rust/rust-${VERSION_NEW}.ebuild" + +popd + +URL="https://github.com/rust-lang/rust/releases/tag/${VERSION_NEW}" + +generate_update_changelog 'Rust' "${VERSION_NEW}" "${URL}" 'rust' + +commit_changes dev-lang/rust "${VERSION_OLD}" "${VERSION_NEW}" \ + profiles \ + virtual/rust + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/rust-release-main.yaml b/.github/workflows/rust-release-main.yaml new file mode 100644 index 0000000000..0617abf9d3 --- /dev/null +++ b/.github/workflows/rust-release-main.yaml @@ -0,0 +1,47 @@ +name: Get the latest Rust release for main +on: + schedule: + - cron: '20 7 * * 2' + workflow_dispatch: + +jobs: + get-rust-release: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + - name: Figure out latest Rust release version + id: rust-latest-release + run: | + version=$(git ls-remote --tags 'https://github.com/rust-lang/rust' | cut -f2 | sed -n "/refs\/tags\/1\.[0-9]*\.[0-9]*$/s/^refs\/tags\///p" | sort -ruV | head -n1) + echo "VERSION_NEW=${version}" >>"${GITHUB_OUTPUT}" + - name: Set up Flatcar SDK + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + CHANNEL: main + run: scripts/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch for main + id: apply-patch-main + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + VERSION_NEW: ${{ steps.rust-latest-release.outputs.VERSION_NEW }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: scripts/.github/workflows/rust-apply-patch.sh + - name: Create pull request for main + id: create-pull-request + uses: peter-evans/create-pull-request@v5 + if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: rust-${{ steps.rust-latest-release.outputs.VERSION_NEW }}-main + base: main + title: Upgrade dev-lang/rust and virtual/rust in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.rust-latest-release.outputs.VERSION_NEW }} + body: Subject says it all. + labels: main diff --git a/.github/workflows/setup-flatcar-sdk.sh b/.github/workflows/setup-flatcar-sdk.sh new file mode 100755 index 0000000000..419b801718 --- /dev/null +++ b/.github/workflows/setup-flatcar-sdk.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -euo pipefail + +if [[ -z "${WORK_SCRIPTS_DIR:-}" ]]; then + echo 'WORK_SCRIPTS_DIR unset, should be pointing to the scripts repo which will be updated' +fi + +sudo ln -sfn /bin/bash /bin/sh +sudo apt-get update +sudo apt-get install -y ca-certificates curl git gnupg lbzip2 lsb-release \ + qemu-user-static +sudo mkdir -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install -y docker-ce docker-ce-cli containerd.io \ + docker-compose-plugin + +pushd "${WORK_SCRIPTS_DIR}" + +source ci-automation/ci_automation_common.sh +source sdk_container/.repo/manifests/version.txt + +# run_sdk_container requires a tag to exist in the repo it resides, +# which may not be the case for forked repos. Add some fake tag in +# this case. +if ! git describe --tags &>/dev/null; then + git tag "${CHANNEL}-${FLATCAR_VERSION}" +fi + +arch="amd64" +sdk_name="flatcar-sdk-${arch}" + +if [[ "${CHANNEL}" = 'main' ]]; then + # for main channel, pull in alpha SDK + MIRROR_LINK='https://alpha.release.flatcar-linux.net/amd64-usr/current' +fi + +# Pin the docker image version to that of the latest release in the channel. +docker_sdk_vernum="$(curl -s -S -f -L "${MIRROR_LINK}/version.txt" \ + | grep -m 1 FLATCAR_SDK_VERSION= | cut -d = -f 2- \ +)" + +docker_image_from_registry_or_buildcache "${sdk_name}" "${docker_sdk_vernum}" + +sdk_full_name="$(docker_image_fullname "${sdk_name}" "${docker_sdk_vernum}")" + +docker_vernum="$(vernum_to_docker_image_version "${FLATCAR_VERSION_ID}")" +packages_container_name="flatcar-packages-${arch}-${docker_vernum}" + +popd + +echo "PACKAGES_CONTAINER=${packages_container_name}" >>"${GITHUB_OUTPUT}" +echo "SDK_NAME=${sdk_full_name}" >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/update-metadata-glsa.yaml b/.github/workflows/update-metadata-glsa.yaml new file mode 100644 index 0000000000..f9b6fb2615 --- /dev/null +++ b/.github/workflows/update-metadata-glsa.yaml @@ -0,0 +1,35 @@ +name: Keep GLSA metadata updated +on: + schedule: + - cron: '0 7 1 * *' + workflow_dispatch: + +jobs: + keep-glsa-metadata-updated: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Update GLSA metadata + id: update-glsa-metadata + run: | + glsa_dir='sdk_container/src/third_party/portage-stable/metadata/glsa' + rm -rf "${glsa_dir}" + mkdir --parents "${glsa_dir}" + rsync --archive rsync://rsync.gentoo.org/gentoo-portage/metadata/glsa/* "${glsa_dir}" + todaydate=$(date +%Y-%m-%d) + echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}" + - name: Create pull request for main branch + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: buildbot/monthly-glsa-metadata-updates-${{steps.update-glsa-metadata.outputs.TODAYDATE }} + delete-branch: true + base: main + title: Monthly GLSA metadata ${{steps.update-glsa-metadata.outputs.TODAYDATE }} + body: Updated GLSA metadata + commit-message: "portage-stable/metadata: Monthly GLSA metadata updates" + author: Flatcar Buildbot + labels: main diff --git a/.github/workflows/update-portage-stable-packages-from-list.yaml b/.github/workflows/update-portage-stable-packages-from-list.yaml new file mode 100644 index 0000000000..bc89cedcd2 --- /dev/null +++ b/.github/workflows/update-portage-stable-packages-from-list.yaml @@ -0,0 +1,92 @@ +name: Keep portage-stable packages updated +on: + schedule: + - cron: '0 7 * * 1' + workflow_dispatch: + +jobs: + keep-packages-updated: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: ./scripts + - name: Check out Gentoo + uses: actions/checkout@v3 + with: + repository: gentoo/gentoo + path: gentoo + # Gentoo is quite a large repo, so limit ourselves to last + # quarter milion of commits. It is about two years worth of changes. + # Is is needed by the sync script to find out the hash of the last commit + # that made the changes to the package. + fetch-depth: 250000 + ref: master + - name: Check out build scripts + uses: actions/checkout@v3 + with: + repository: flatcar/flatcar-build-scripts + path: flatcar-build-scripts + - name: Update listed packages + id: update-listed-packages + run: | + git config --global user.name "Flatcar Buildbot" + git config --global user.email "buildbot@flatcar-linux.org" + old_head=$(git -C scripts rev-parse HEAD) + packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list) + gentoo_repo=$(realpath gentoo) + build_scripts=$(realpath flatcar-build-scripts) + pushd scripts/sdk_container/src/third_party/portage-stable + while read -r package; do + if [[ ! -e "${package}" ]]; then + # If this happens, it means that the package was moved to overlay + # or dropped, the list ought to be updated. + echo "::warning title=${package}::Nonexistent package" + continue + fi + if [[ ! -e "${gentoo_repo}/${package}" ]]; then + # If this happens, it means that the package was obsoleted or moved + # in Gentoo. The obsoletion needs to be handled in the case-by-case + # manner, while move should be handled by doing the same move + # in portage-stable. The build should not break because of the move, + # because most likely it's already reflected in the profiles/updates + # directory. + echo "::warning title=${package}::Obsolete or moved package" + continue + fi + GENTOO_REPO="${gentoo_repo}" "${build_scripts}/sync-with-gentoo" "${package}" + done < <(grep '^[^#]' "${packages_list}") + popd + new_head=$(git -C scripts rev-parse HEAD) + updated=0 + if [[ "${new_head}" != "${old_head}" ]]; then + updated=1 + fi + todaydate=$(date +%Y-%m-%d) + echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}" + echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}" + - name: Create pull request for main branch + uses: peter-evans/create-pull-request@v5 + if: steps.update-listed-packages.outputs.UPDATED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }} + delete-branch: true + base: main + title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }} + body: | + CI: TODO + + -- + + TODO: Changes. + + -- + + - [ ] changelog + - [ ] image diff + labels: main + draft: true diff --git a/.github/workflows/vmware-apply-patch.sh b/.github/workflows/vmware-apply-patch.sh new file mode 100755 index 0000000000..b9b1585552 --- /dev/null +++ b/.github/workflows/vmware-apply-patch.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -euo pipefail + +source "${GHA_SCRIPTS_DIR}/.github/workflows/common.sh" + +prepare_git_repo + +# Update app-emulation/open-vm-tools + +pushd "${SDK_OUTER_OVERLAY}" + +# Parse the Manifest file for already present source files and keep the latest version in the current series +VERSION_OLD=$(sed -n "s/^DIST open-vm-tools-\([0-9]*\.[0-9]*\.[0-9]*\).*$/\1/p" app-emulation/open-vm-tools/Manifest | sort -ruV | head -n1) +if [[ "${VERSION_NEW}" = "${VERSION_OLD}" ]]; then + echo "already the latest open-vm-tools, nothing to do" + exit 0 +fi + +EBUILD_FILENAME_OVT=$(get_ebuild_filename app-emulation/open-vm-tools "${VERSION_OLD}") +git mv "${EBUILD_FILENAME_OVT}" "app-emulation/open-vm-tools/open-vm-tools-${VERSION_NEW}.ebuild" + +# We need to also replace the old build number with the new build number in the ebuild. +sed -i -e "s/^\(MY_P=.*-\)[0-9]*\"$/\1${BUILD_NUMBER}\"/" "app-emulation/open-vm-tools/open-vm-tools-${VERSION_NEW}.ebuild" + +# Also update coreos-base/oem-vmware +EBUILD_FILENAME_OEM=$(get_ebuild_filename coreos-base/oem-vmware "${VERSION_OLD}") +git mv "${EBUILD_FILENAME_OEM}" "coreos-base/oem-vmware/oem-vmware-${VERSION_NEW}.ebuild" + +popd + +URL="https://github.com/vmware/open-vm-tools/releases/tag/stable-${VERSION_NEW}" + +generate_update_changelog 'open-vm-tools' "${VERSION_NEW}" "${URL}" 'open-vm-tools' + +commit_changes app-emulation/open-vm-tools "${VERSION_OLD}" "${VERSION_NEW}" \ + coreos-base/oem-vmware + +cleanup_repo + +echo "VERSION_OLD=${VERSION_OLD}" >>"${GITHUB_OUTPUT}" +echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}" diff --git a/.github/workflows/vmware-release-main.yaml b/.github/workflows/vmware-release-main.yaml new file mode 100644 index 0000000000..03e6d6cd03 --- /dev/null +++ b/.github/workflows/vmware-release-main.yaml @@ -0,0 +1,49 @@ +name: Get the latest open-vm-tools release for main +on: + schedule: + - cron: '0 7 * * 3' + workflow_dispatch: + +jobs: + get-vmware-release: + runs-on: ubuntu-latest + steps: + - name: Check out scripts + uses: actions/checkout@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + - name: Figure out latest open-vm-tools release version + id: openvmtools-latest-release + run: | + version=$(git ls-remote --tags 'https://github.com/vmware/open-vm-tools' | cut -f2 | sed -n "/refs\/tags\/stable-[0-9]*\.[0-9]*\.[0-9]*$/s/^refs\/tags\/stable-//p" | sort -ruV | head -n1) + buildNumber=$(curl -sSL https://api.github.com/repos/vmware/open-vm-tools/releases/latest | jq -r '.assets[0].name' | sed -n "s/^open-vm-tools-${version}*-\([0-9]*\)\..*/\1/p") + echo "BUILD_NUMBER=${buildNumber}" >>"${GITHUB_OUTPUT}" + echo "VERSION_NEW=${version}" >>"${GITHUB_OUTPUT}" + - name: Set up Flatcar SDK + id: setup-flatcar-sdk + env: + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + CHANNEL: main + run: scripts/.github/workflows/setup-flatcar-sdk.sh + - name: Apply patch for main + id: apply-patch-main + env: + GHA_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts" + BUILD_NUMBER: ${{ steps.openvmtools-latest-release.outputs.BUILD_NUMBER }} + VERSION_NEW: ${{ steps.openvmtools-latest-release.outputs.VERSION_NEW }} + PACKAGES_CONTAINER: ${{ steps.setup-flatcar-sdk.outputs.PACKAGES_CONTAINER }} + SDK_NAME: ${{ steps.setup-flatcar-sdk.outputs.SDK_NAME }} + run: scripts/.github/workflows/vmware-apply-patch.sh + - name: Create pull request for main + uses: peter-evans/create-pull-request@v5 + if: steps.apply-patch-main.outputs.UPDATE_NEEDED == 1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + path: scripts + branch: vmware-${{ steps.openvmtools-latest-release.outputs.VERSION_NEW }}-main + base: main + title: Upgrade open-vm-tools in main from ${{ steps.apply-patch-main.outputs.VERSION_OLD }} to ${{ steps.openvmtools-latest-release.outputs.VERSION_NEW }} + body: Subject says it all. + labels: main