From 25dbccc14dc115a0ceb7542d6274d8a3bd4f9191 Mon Sep 17 00:00:00 2001 From: Kai Lueke Date: Wed, 26 Oct 2022 15:46:15 +0200 Subject: [PATCH] ci-automation: Support local patches For embargoed releases it is useful to apply patches locally to build with them before they are public. This allows to push the same patches to the repo during the Flatcar release at the embargo lift. The result is the same (as long as the scripts patches did not change parts of the setup logic that was running before they got applied), we can just build earlier and thus do the Flatcar release directly on the embargo lift instead of having to wait with the build because it would require the patches to be in the repos. --- ci-automation/README.md | 4 ++++ ci-automation/ci_automation_common.sh | 32 +++++++++++++++++++++++++++ ci-automation/image.sh | 7 ++++++ ci-automation/packages-tag.sh | 7 ++++++ ci-automation/packages.sh | 7 ++++++ ci-automation/sdk_bootstrap.sh | 7 ++++++ ci-automation/vms.sh | 8 +++++++ 7 files changed, 72 insertions(+) diff --git a/ci-automation/README.md b/ci-automation/README.md index 13187e7afb..9e84e35ef1 100644 --- a/ci-automation/README.md +++ b/ci-automation/README.md @@ -112,6 +112,10 @@ image_build amd64 `- vendor OS images ---->| ``` +## Local Patches + +For embargoed relases the build system looks for patch files `../scripts.patch`, `../overlay.patch`, `../portage.patch` (i.e., in the folder that contains the `scripts` repo) and applies them locally before building. + ## Testing Testing follows the same design principles build automation adheres to - it's self-contained and context-aware, reducing required parameters to a minimum. diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 50f7fbcd75..e65816086f 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -475,3 +475,35 @@ function list_files() { done } # -- + +# Looks for ../scripts.patch, ../overlay.patch, ../portage.patch and +# applies them to the current repo or the respective sub-module checkout. +function apply_local_patches() { + local patch_files=(../scripts.patch ../overlay.patch ../portage.patch) + local patch_file + local patch_id + local dirarg + echo "Looking for local patches ${patch_files[*]}" + for patch_file in "${patch_files[@]}"; do + if [ "${patch_file}" = "../scripts.patch" ]; then + dirarg=() + elif [ "${patch_file}" = "../overlay.patch" ]; then + dirarg=("-C" "sdk_container/src/third_party/coreos-overlay/") + elif [ "${patch_file}" = "../portage.patch" ]; then + dirarg=("-C" "sdk_container/src/third_party/portage-stable/") + else + echo "wrong case: unexpected ${patch_file}" + exit 1 + fi + patch_id=$(test -e "${patch_file}" && { cat "${patch_file}" | git patch-id | cut -d ' ' -f 1 ; } || true) + if [ "${patch_id}" != "" ]; then + if git "${dirarg[@]}" log --no-merges -p HEAD | git patch-id | cut -d ' ' -f 1 | grep -q "${patch_id}"; then + echo "Skipping already applied ${patch_file}" + else + echo "Applying ${patch_file}" + GIT_COMMITTER_NAME="Flatcar Buildbot" GIT_COMMITTER_EMAIL="buildbot@flatcar-linux.org" git "${dirarg[@]}" am -3 "$PWD/${patch_file}" + fi + fi + done +} +# -- diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 9a69de92c8..12328e5301 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -33,6 +33,12 @@ # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. # +# 3. A file ../scripts.patch to apply with "git am -3" for the scripts repo. +# +# 4. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. +# +# 5. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. +# # OUTPUT: # # 1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache. @@ -83,6 +89,7 @@ function _image_build_impl() { official_arg="--noofficial" fi + apply_local_patches # build image and related artifacts ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ diff --git a/ci-automation/packages-tag.sh b/ci-automation/packages-tag.sh index 562e23efc3..15d1c7d501 100644 --- a/ci-automation/packages-tag.sh +++ b/ci-automation/packages-tag.sh @@ -42,6 +42,12 @@ # This version will be checked out / pulled from remote in the portage-stable git submodule. # The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. # +# 4. A file ../scripts.patch to apply with "git am -3" for the scripts repo. +# +# 5. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. +# +# 6. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. +# # OUTPUT: # # 1. Updated scripts repository @@ -120,5 +126,6 @@ function _packages_tag_impl() { create_versionfile "$sdk_version" "$version" ) update_and_push_version "${version}" "${push_branch}" + apply_local_patches } # -- diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 465eb4d58e..871475b7e1 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -36,6 +36,12 @@ # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. # +# 4. A file ../scripts.patch to apply with "git am -3" for the scripts repo. +# +# 5. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. +# +# 6. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. +# # OUTPUT: # # 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages @@ -93,6 +99,7 @@ function _packages_build_impl() { torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx" fi + apply_local_patches # Build packages; store packages and torcx output in container ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ -C "${sdk_image}" \ diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 81867f9d2e..c793b268c4 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -47,6 +47,12 @@ # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. # +# 8. A file ../scripts.patch to apply with "git am -3" for the scripts repo. +# +# 9. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. +# +# 10. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. +# # OUTPUT: # # 1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache. @@ -143,6 +149,7 @@ function _sdk_bootstrap_impl() { create_versionfile "${vernum}" ) update_and_push_version "${version}" "${push_branch}" + apply_local_patches ./bootstrap_sdk_container -x ./ci-cleanup.sh "${seed_version}" "${vernum}" diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 60765b9bd3..1e6cdd43f4 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -36,6 +36,12 @@ # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. # +# 3. A file ../scripts.patch to apply with "git am -3" for the scripts repo. +# +# 4. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. +# +# 5. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. +# # OUTPUT: # # 1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ ) @@ -76,6 +82,8 @@ function _vm_build_impl() { local vms="flatcar-vms-${arch}" local vms_container="${vms}-${docker_vernum}" + apply_local_patches + # automatically add PXE to formats if we build for Equinix Metal (packet). local has_packet=0 local has_pxe=0