From 992db543d101d5ac50e8dfae70008b59468e0a8f Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Wed, 20 Sep 2023 17:54:58 +0200
Subject: [PATCH 01/32] ci-automation/image-changes, .github/ci: Drop work
 directory parameter

It was only needed for the show-changes script. Now that show-changes
script allows to set the repos parent directory with an environment
variable, we set the variable instead of changing the working
directory.
---
 .github/workflows/ci.yaml      | 10 ++--------
 ci-automation/image_changes.sh | 24 ++++--------------------
 2 files changed, 6 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5512673032..3e8a55d289 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -216,10 +216,7 @@ jobs:
               "local:${PWD}/artifacts/${arch}-usr/latest"
           )
           show_changes_env=(
-              # Nothing to add.
-              "SCRIPTS_REPO=scripts"
-              "COREOS_OVERLAY_REPO=coreos-overlay"
-              "PORTAGE_STABLE_REPO=portage-stable"
+              "REPOS_PARENT_DIR=.."
           )
           show_changes_params_overrides=(
               # We may not have a tag handy, so we tell show-changes
@@ -228,11 +225,8 @@ jobs:
               'NEW_VERSION=HEAD'
           )
 
-          # Parent directory of the scripts repo, required by some other
-          # script.
-          work_directory='..'
           generate_image_changes_report \
-              "${arch}" "${channel}" "${vernum}" 'image-changes-reports.txt' "../flatcar-build-scripts" "${work_directory}" \
+              "${arch}" "${channel}" "${vernum}" 'image-changes-reports.txt' "../flatcar-build-scripts" \
               "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
               "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
               "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}"
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 82d9857c7e..954f6c8830 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -63,9 +63,7 @@ function image_changes() (
     show_changes_env=(
         # Provide a python3 command for the CVE DB parsing
         "PATH=${PATH}:${PWD}/ci-automation/python-bin"
-        "SCRIPTS_REPO=scripts"
-        "COREOS_OVERLAY_REPO=coreos-overlay"
-        "PORTAGE_STABLE_REPO=portage-stable"
+        "REPOS_PARENT_DIR=.."
     )
     show_changes_params_overrides=(
         # Nothing to override.
@@ -77,16 +75,13 @@ function image_changes() (
         --depth 1 \
         "https://github.com/flatcar/flatcar-build-scripts" \
         "${fbs_repo}"
-    # Parent directory of the scripts repo, required by some other
-    # script.
-    local work_directory='..'
     if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then
         local BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}")
     fi
     echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/flatcar_production_image.bin.bz2"
     echo
     generate_image_changes_report \
-        "${arch}" "${channel}" "${vernum}" '-' "${fbs_repo}" "${work_directory}" \
+        "${arch}" "${channel}" "${vernum}" '-' "${fbs_repo}" \
         "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
         "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
         "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}"
@@ -98,7 +93,6 @@ function image_changes() (
 # 3 - version (FLATCAR_VERSION)
 # 4 - report file (can be relative)
 # 5 - flatcar-build-scripts directory (can be relative, will be realpathed)
-# 6 - work directory for the report scripts (must be a parent directory of the scripts repo, can be relative)
 # @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides
 #
 # Example:
@@ -116,7 +110,6 @@ function generate_image_changes_report() (
     local vernum=${1}; shift
     local report_output=${1}; shift
     local flatcar_build_scripts_repo=${1}; shift
-    local work_directory=${1}; shift
 
     local -a package_diff_env package_diff_params
     local -a size_changes_env size_changes_params
@@ -182,7 +175,7 @@ function generate_image_changes_report() (
 
     local print_image_reports_invocation=(
         print_image_reports
-        "${flatcar_build_scripts_repo}" "${channel_a}" "${version_a}" "${work_directory}"
+        "${flatcar_build_scripts_repo}" "${channel_a}" "${version_a}"
         "${package_diff_env[@]}" --- "${package_diff_params[@]}" --
         "${size_changes_env[@]}" --- "${size_changes_params[@]}" --
         "${show_changes_env[@]}" --- "${show_changes_params[@]}"
@@ -256,7 +249,7 @@ function channel_version() {
 # flatcar-build-scripts repo. The environment and parameters for the
 # scripts are passed as follows:
 #
-# print_image_reports <flatcar-build-scripts-directory> <channel a> <version a> <work dir> \\
+# print_image_reports <flatcar-build-scripts-directory> <channel a> <version a> \\
 #       <env vars for package-diff> --- <parameters for package-diff> -- \\
 #       <env vars for size-change-report.sh> --- <parameters for size-change-report.sh> -- \\
 #       <env vars for show-changes> --- <parameters for show-changes>
@@ -284,7 +277,6 @@ function print_image_reports() {
     local flatcar_build_scripts_repo=${1}; shift
     local channel_a=${1}; shift
     local version_a=${1}; shift
-    local work_directory=${1}; shift
     local -a package_diff_env=() package_diff_params=()
     local -a size_change_report_env=() size_change_report_params=()
     local -a show_changes_env=() show_changes_params=()
@@ -309,32 +301,27 @@ function print_image_reports() {
     echo "== Image differences compared to ${channel_a} ${version_a} =="
     echo "Package updates, compared to ${channel_a} ${version_a}:"
     env \
-        --chdir="${work_directory}" \
         "${package_diff_env[@]}" FILE=flatcar_production_image_packages.txt \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
     echo "Image file changes, compared to ${channel_a} ${version_a}:"
     env \
-        --chdir="${work_directory}" \
         "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
     echo "Image kernel config changes, compared to ${channel_a} ${version_a}:"
     env \
-        --chdir="${work_directory}" \
         "${package_diff_env[@]}" FILE=flatcar_production_image_kernel_config.txt \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
     echo "Image init ramdisk file changes, compared to ${channel_a} ${version_a}:"
     env \
-        --chdir="${work_directory}" \
         "${package_diff_env[@]}" FILE=flatcar_production_image_initrd_contents.txt FILESONLY=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
 
     local size_changes_invocation=(
         env
-        --chdir="${work_directory}"
         "${size_change_report_env[@]}"
         "${flatcar_build_scripts_repo}/size-change-report.sh"
     )
@@ -354,7 +341,6 @@ function print_image_reports() {
 
     echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${channel_a} ${version_a}:"
     env \
-        --chdir="${work_directory}" \
         "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt CALCSIZE=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
@@ -365,7 +351,6 @@ function print_image_reports() {
     done
     # The first changelog we print is always against the previous version of the new channel (is only same as ${channel_a} ${version_a} without a transition)
     env \
-        --chdir "${work_directory}" \
         "${show_changes_env[@]}" \
         "${flatcar_build_scripts_repo}/show-changes" \
         "${SHOW_CHANGES_NEW_CHANNEL}-${SHOW_CHANGES_NEW_CHANNEL_PREV_VERSION}" \
@@ -373,7 +358,6 @@ function print_image_reports() {
     # See if a channel transition happened and print the changelog against ${channel_a} ${version_a} which is the previous release
     if [ "${channel_a}" != "${SHOW_CHANGES_NEW_CHANNEL}" ]; then
         env \
-            --chdir "${work_directory}" \
             "${show_changes_env[@]}" \
             "${flatcar_build_scripts_repo}/show-changes" \
             "${channel_a}-${version_a}" \

From 5443a66c9172211dd7962e98a47cf8e15622f7bb Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Wed, 20 Sep 2023 17:58:15 +0200
Subject: [PATCH 02/32] ci-automation/image-changes, .github/ci: Show changes
 in OEM sysext images

---
 .github/workflows/ci.yaml      |   5 +-
 ci-automation/image_changes.sh | 107 ++++++++++++++++++++++++++-------
 2 files changed, 90 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 3e8a55d289..106f0b7853 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -225,11 +225,14 @@ jobs:
               'NEW_VERSION=HEAD'
           )
 
+          declare -a oemids
+          get_oem_id_list . oemids
           generate_image_changes_report \
               "${arch}" "${channel}" "${vernum}" 'image-changes-reports.txt' "../flatcar-build-scripts" \
               "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
               "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
-              "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}"
+              "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}" -- \
+              "${oemids[@]}"
 
       - name: Build VM image(s)
         shell: bash
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 954f6c8830..b3fdb57ffd 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -80,20 +80,56 @@ function image_changes() (
     fi
     echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/flatcar_production_image.bin.bz2"
     echo
+    local -a oemids
+    get_oem_id_list . oemids
     generate_image_changes_report \
         "${arch}" "${channel}" "${vernum}" '-' "${fbs_repo}" \
         "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
         "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
-        "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}"
+        "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}" -- \
+        "${oemids[@]}"
 )
 # --
 
+function get_oem_id_list() {
+    local scripts_repo
+    scripts_repo=${1}; shift
+    local -n list_var_ref=${1}; shift
+
+    local -a ebuilds
+    ebuilds=( "${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-"*'.ebuild' )
+
+    list_var_ref=()
+    if [[ ${#ebuilds[@]} -eq 0 ]]; then
+        return 0
+    fi
+    local line mode
+    # 0 = none OEMIDS line found yet
+    # 1 = OEMIDS line found
+    mode=0
+    while read -r line; do
+        case ${mode} in
+            0)
+                if [[ ${line} = 'OEMIDS=(' ]]; then
+                    mode=1
+                fi
+                ;;
+            1)
+                if [[ ${line} = ')' ]]; then
+                    break
+                fi
+                list_var_ref+=( "${line}" )
+                ;;
+        esac
+    done <"${ebuilds[0]}"
+}
+
 # 1 - arch
 # 2 - channel (alpha, beta, stable or lts)
 # 3 - version (FLATCAR_VERSION)
 # 4 - report file (can be relative)
 # 5 - flatcar-build-scripts directory (can be relative, will be realpathed)
-# @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides
+# @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides -- list of OEM ids
 #
 # Example:
 #
@@ -101,7 +137,7 @@ function image_changes() (
 #     amd64 alpha 3456.0.0+my-changes reports/images.txt ../flatcar-build-scripts .. \\
 #     FROM_B=bincache BOARD_B=amd64-usr --- 3456.0.0+my-changes -- \\
 #     --- bincache:amd64:3456.0.0+my-changes -- \\
-#     "PATH=${PATH}:${PWD}/ci-automation/python-bin"
+#     "PATH=${PATH}:${PWD}/ci-automation/python-bin" --- -- azure vmware
 function generate_image_changes_report() (
     set -euo pipefail
 
@@ -114,6 +150,7 @@ function generate_image_changes_report() (
     local -a package_diff_env package_diff_params
     local -a size_changes_env size_changes_params
     local -a show_changes_env show_changes_params
+    local -a oemids
     local params_shift=0
 
     split_to_env_and_params \
@@ -127,6 +164,8 @@ function generate_image_changes_report() (
     split_to_env_and_params \
         show_changes_env show_changes_params params_shift \
         "${@}"
+    shift "${params_shift}"
+    oemids=( "${@}" )
 
     local new_channel new_channel_prev_version channel_a version_a
     local board="${arch}-usr"
@@ -178,7 +217,8 @@ function generate_image_changes_report() (
         "${flatcar_build_scripts_repo}" "${channel_a}" "${version_a}"
         "${package_diff_env[@]}" --- "${package_diff_params[@]}" --
         "${size_changes_env[@]}" --- "${size_changes_params[@]}" --
-        "${show_changes_env[@]}" --- "${show_changes_params[@]}"
+        "${show_changes_env[@]}" --- "${show_changes_params[@]}" --
+        "${oemids[@]}"
     )
     # Using "|| :" to avoid failing the job.
     if [[ ${report_output} = '-' ]]; then
@@ -252,7 +292,8 @@ function channel_version() {
 # print_image_reports <flatcar-build-scripts-directory> <channel a> <version a> \\
 #       <env vars for package-diff> --- <parameters for package-diff> -- \\
 #       <env vars for size-change-report.sh> --- <parameters for size-change-report.sh> -- \\
-#       <env vars for show-changes> --- <parameters for show-changes>
+#       <env vars for show-changes> --- <parameters for show-changes> -- \\
+#       <list of OEM ids>
 #
 # Env vars are passed to the called scripts verbatim. Parameters are
 # described below.
@@ -280,6 +321,7 @@ function print_image_reports() {
     local -a package_diff_env=() package_diff_params=()
     local -a size_change_report_env=() size_change_report_params=()
     local -a show_changes_env=() show_changes_params=()
+    local -a oemids
     local params_shift=0
 
     split_to_env_and_params \
@@ -293,11 +335,19 @@ function print_image_reports() {
     split_to_env_and_params \
         show_changes_env show_changes_params params_shift \
         "${@}"
+    shift "${params_shift}"
+    oemids=( "${@}" )
 
     flatcar_build_scripts_repo=$(realpath "${flatcar_build_scripts_repo}")
 
     echo "==================================================================="
 
+    local size_changes_invocation=(
+        env
+        "${size_change_report_env[@]}"
+        "${flatcar_build_scripts_repo}/size-change-report.sh"
+    )
+
     echo "== Image differences compared to ${channel_a} ${version_a} =="
     echo "Package updates, compared to ${channel_a} ${version_a}:"
     env \
@@ -309,27 +359,28 @@ function print_image_reports() {
         "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
+    echo "Image file size changes, compared to ${channel_a} ${version_a}:"
+    if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:wtd}" 2>&1; then
+        "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:old}" 2>&1
+    fi
+    echo
     echo "Image kernel config changes, compared to ${channel_a} ${version_a}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_kernel_config.txt \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
+    echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${channel_a} ${version_a}:"
+    env \
+        "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt CALCSIZE=1 \
+        "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
+    echo
+
+    echo "== Init ramdisk differences compared to ${channel_a} ${version_a} =="
     echo "Image init ramdisk file changes, compared to ${channel_a} ${version_a}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_initrd_contents.txt FILESONLY=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
-
-    local size_changes_invocation=(
-        env
-        "${size_change_report_env[@]}"
-        "${flatcar_build_scripts_repo}/size-change-report.sh"
-    )
-    echo "Image file size changes, compared to ${channel_a} ${version_a}:"
-    if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:wtd}" 2>&1; then
-        "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:old}" 2>&1
-    fi
-    echo
     echo "Image init ramdisk file size changes, compared to ${channel_a} ${version_a}:"
     if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-wtd}" 2>&1; then
         "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-old}" 2>&1
@@ -339,11 +390,25 @@ function print_image_reports() {
     echo "Note that vmlinuz-a also contains the kernel code, which might have changed too, so the reported difference does not accurately describe the change in initrd."
     echo
 
-    echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${channel_a} ${version_a}:"
-    env \
-        "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt CALCSIZE=1 \
-        "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-    echo
+    local oemid
+    for oemid in "${oemids[@]}"; do
+        echo "== Sysext changes for OEM ${oemid} compared to ${channel_a} ${version_a} =="
+        echo "Package updates, compared to ${channel_a} ${version_a}:"
+        env \
+            "${package_diff_env[@]}" FILE="oem-${oemid}_packages.txt" \
+            "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
+        echo
+        echo "Image file changes, compared to ${channel_a} ${version_a}:"
+        env \
+            "${package_diff_env[@]}" FILE="oem-${oemid}_contents.txt" FILESONLY=1 CUTKERNEL=1 \
+            "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
+        echo
+        echo "Image file size changes, compared to ${channel_a} ${version_a}:"
+        if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-wtd}"; then
+            "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-old}" 2>&1
+        fi
+        echo
+    done
 
     local param
     for param in "${show_changes_params[@]}"; do

From ffa2bfc6953b2c0af174c8824fcfc220df6de9eb Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 21 Sep 2023 10:37:40 +0200
Subject: [PATCH 03/32] ci-automation/image-changes, .github/ci: Set up env for
 show-changes

---
 .github/workflows/ci.yaml      | 4 +++-
 ci-automation/image_changes.sh | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 106f0b7853..39fb474204 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -216,7 +216,9 @@ jobs:
               "local:${PWD}/artifacts/${arch}-usr/latest"
           )
           show_changes_env=(
-              "REPOS_PARENT_DIR=.."
+              "SCRIPTS_REPO=."
+              "COREOS_OVERLAY_REPO=../coreos-overlay"
+              "PORTAGE_STABLE_REPO=../portage-stable"
           )
           show_changes_params_overrides=(
               # We may not have a tag handy, so we tell show-changes
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index b3fdb57ffd..a6c9b0920e 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -63,7 +63,9 @@ function image_changes() (
     show_changes_env=(
         # Provide a python3 command for the CVE DB parsing
         "PATH=${PATH}:${PWD}/ci-automation/python-bin"
-        "REPOS_PARENT_DIR=.."
+        "SCRIPTS_REPO=."
+        "COREOS_OVERLAY_REPO=../coreos-overlay"
+        "PORTAGE_STABLE_REPO=../portage-stable"
     )
     show_changes_params_overrides=(
         # Nothing to override.

From 5915caff2138cc78de68a3821c5b1d9ab24da50a Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 21 Sep 2023 10:40:51 +0200
Subject: [PATCH 04/32] ci-automation/image-changes: Set NEW_VERSION in
 toplevel function

This avoid assuming that the current working directory in the
generate_image_changes_report function is actually toplevel directory
of the scripts repo.
---
 ci-automation/image_changes.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index a6c9b0920e..2daaa87292 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -68,7 +68,9 @@ function image_changes() (
         "PORTAGE_STABLE_REPO=../portage-stable"
     )
     show_changes_params_overrides=(
-        # Nothing to override.
+        # The show-changes script expects a tag name, so using git tag
+        # here instead of the vernum variable.
+        "NEW_VERSION=$(git tag --points-at HEAD)"
     )
 
     local fbs_repo='../flatcar-build-scripts'
@@ -205,9 +207,6 @@ function generate_image_changes_report() (
 
     # Nothing to prepend to show_changes_env.
     show_changes_params=(
-        # The show-changes script expects a tag name, so using git tag
-        # here instead of the vernum variable.
-        "NEW_VERSION=$(git tag --points-at HEAD)"
         "NEW_CHANNEL=${new_channel}"
         "NEW_CHANNEL_PREV_VERSION=${new_channel_prev_version}"
         # Potential overrides.

From f9e0363bf05202aa14f3dd2f5a246970527dffee Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 21 Sep 2023 16:12:17 +0200
Subject: [PATCH 05/32] ci-automation/image-changes, .github/ci: Add reports
 against last nightly

---
 .github/workflows/ci.yaml      |  95 ++++++--
 ci-automation/image_changes.sh | 399 ++++++++++++++++++++++-----------
 2 files changed, 353 insertions(+), 141 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 39fb474204..7826195c24 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -187,7 +187,7 @@ jobs:
                                 --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
                                 prodtar container
 
-      - name: Generate reports
+      - name: Generate reports against last release
         shell: bash
         run: |
           set -euo pipefail
@@ -195,24 +195,31 @@ jobs:
 
           source ci-automation/image_changes.sh
 
-          channel=alpha
-          vernum=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
-          board="${arch}-usr"
+          git_tag=''
+          git_tag_for_release . git_tag
+          declare -a var_names=(
+              package_diff_env package_diff_params
+              size_changes_env size_changes_params
+              show_changes_env show_changes_params
+          )
+          declare -a "${var_names[@]}"
+          version_description=''
+          var_names+=( version_description )
 
-          package_diff_env=(
+          prepare_env_vars_and_params_for_release "${arch}" "${git_tag}" "${var_names[@]}"
+
+          package_diff_env+=(
               "FROM_B=file://${PWD}/artifacts/${arch}-usr/latest"
               # BOARD_B and CHANNEL_B are unused.
           )
-          package_diff_params_b=(
+          package_diff_params+=(
               # The package-diff script appends version to the file
               # URL, but the directory with the image has no version
               # component at its end, so we use . as a version.
               '.'
           )
-          size_changes_env=(
-              # Nothing to add.
-          )
-          size_changes_params_b=(
+          # Nothing to add to size changes env.
+          size_changes_params+=(
               "local:${PWD}/artifacts/${arch}-usr/latest"
           )
           show_changes_env=(
@@ -220,7 +227,7 @@ jobs:
               "COREOS_OVERLAY_REPO=../coreos-overlay"
               "PORTAGE_STABLE_REPO=../portage-stable"
           )
-          show_changes_params_overrides=(
+          show_changes_params=(
               # We may not have a tag handy, so we tell show-changes
               # to use git HEAD as a reference to new changelog
               # entries.
@@ -230,10 +237,66 @@ jobs:
           declare -a oemids
           get_oem_id_list . oemids
           generate_image_changes_report \
-              "${arch}" "${channel}" "${vernum}" 'image-changes-reports.txt' "../flatcar-build-scripts" \
-              "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
-              "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
-              "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}" -- \
+              "${version_description}" 'image-changes-reports-release.txt' "../flatcar-build-scripts" \
+              "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
+              "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
+              "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
+              "${oemids[@]}"
+
+      - name: Generate reports against last nightly
+        shell: bash
+        run: |
+          set -euo pipefail
+          set -x
+
+          source ci-automation/image_changes.sh
+
+          git_tag=''
+          git_tag_for_nightly . git_tag
+          declare -a var_names=(
+              package_diff_env package_diff_params
+              size_changes_env size_changes_params
+              show_changes_env show_changes_params
+          )
+          declare -a "${var_names[@]}"
+          version_description=''
+          var_names+=( version_description )
+
+          prepare_env_vars_and_params_for_bincache "${arch}" "${git_tag}" "${var_names[@]}"
+
+          package_diff_env+=(
+              "FROM_B=file://${PWD}/artifacts/${arch}-usr/latest"
+              # BOARD_B and CHANNEL_B are unused.
+          )
+          package_diff_params+=(
+              # The package-diff script appends version to the file
+              # URL, but the directory with the image has no version
+              # component at its end, so we use . as a version.
+              '.'
+          )
+          # Nothing to add to size changes env.
+          size_changes_params+=(
+              "local:${PWD}/artifacts/${arch}-usr/latest"
+          )
+          show_changes_env=(
+              "SCRIPTS_REPO=."
+              "COREOS_OVERLAY_REPO=../coreos-overlay"
+              "PORTAGE_STABLE_REPO=../portage-stable"
+          )
+          show_changes_params=(
+              # We may not have a tag handy, so we tell show-changes
+              # to use git HEAD as a reference to new changelog
+              # entries.
+              'NEW_VERSION=HEAD'
+          )
+
+          declare -a oemids
+          get_oem_id_list . oemids
+          generate_image_changes_report \
+              "${version_description}" 'image-changes-reports-nightly.txt' "../flatcar-build-scripts" \
+              "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
+              "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
+              "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
               "${oemids[@]}"
 
       - name: Build VM image(s)
@@ -327,7 +390,7 @@ jobs:
           retention-days: 7
           name: ${{ matrix.arch }}-image-changes-reports
           path: |
-            scripts/image-changes-reports.txt
+            scripts/image-changes-reports*.txt
 
       # Clean up what we uploaded already so the "vendor images" wildcard
       #  works when uploading artifacts in the next step.
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 2daaa87292..13691d6860 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -19,6 +19,7 @@
 # INPUT:
 #
 #   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
+#   2. What to compare against, can be "last-release" or "last-nightly".
 #
 # OPTIONAL INPUT:
 #
@@ -33,41 +34,60 @@
 function image_changes() (
     set -euo pipefail
 
-    local arch=${1}
-    local channel vernum
+    local arch=${1}; shift
+    local what=${1-last-release}; shift
 
-    channel=$(source sdk_lib/sdk_container_common.sh; get_git_channel)
-    if [ "${channel}" = "developer" ]; then
-            channel="alpha"
-    fi
-    vernum=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
+    local -a package_diff_env package_diff_params
+    local -a size_changes_env size_changes_params
+    local -a show_changes_env show_changes_params
+    local version_description
+    local -a var_names=(
+        package_diff_env package_diff_params
+        size_changes_env size_changes_params
+        show_changes_env show_changes_params
+        version_description
+    )
 
-    local -a package_diff_env package_diff_params_b
-    local -a size_changes_env size_changes_params_b
-    local -a show_changes_env show_changes_params_overrides
+    case ${what} in
+        last-release)
+            local git_tag
+            git_tag_for_release . git_tag
+            prepare_env_vars_and_params_for_release "${arch}" "${git_tag}" "${var_names[@]}"
+            ;;
+        last-nightly)
+            local git_tag
+            git_tag_for_nightly . git_tag
+            prepare_env_vars_and_params_for_bincache "${arch}" "${git_tag}" "${var_names[@]}"
+            ;;
+        *)
+            echo "invalid argument '${what}', expected 'last-nightly' or 'last-release'" >&2
+            exit 1
+            ;;
+    esac
 
-    package_diff_env=(
+    local version
+    version=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
+    package_diff_env+=(
         "FROM_B=bincache"
         "BOARD_B=${arch}-usr"
         # CHANNEL_B is unused
     )
-    package_diff_params_b=(
-        "${vernum}"
+    package_diff_params+=(
+        "${version}"
     )
-    size_changes_env=(
-        # Nothing to add.
+    # Nothing to add to size changes env.
+    size_changes_params+=(
+        "bincache:${arch}:${version}"
     )
-    size_changes_params_b=(
-        "bincache:${arch}:${vernum}"
-    )
-    show_changes_env=(
+    show_changes_env+=(
         # Provide a python3 command for the CVE DB parsing
         "PATH=${PATH}:${PWD}/ci-automation/python-bin"
+        # Override the default locations of repositories.
         "SCRIPTS_REPO=."
         "COREOS_OVERLAY_REPO=../coreos-overlay"
         "PORTAGE_STABLE_REPO=../portage-stable"
     )
-    show_changes_params_overrides=(
+    show_changes_params+=(
         # The show-changes script expects a tag name, so using git tag
         # here instead of the vernum variable.
         "NEW_VERSION=$(git tag --points-at HEAD)"
@@ -82,19 +102,56 @@ function image_changes() (
     if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then
         local BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}")
     fi
-    echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/flatcar_production_image.bin.bz2"
+    echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${version}/flatcar_production_image.bin.bz2"
     echo
     local -a oemids
     get_oem_id_list . oemids
     generate_image_changes_report \
-        "${arch}" "${channel}" "${vernum}" '-' "${fbs_repo}" \
-        "${package_diff_env[@]}" --- "${package_diff_params_b[@]}" -- \
-        "${size_changes_env[@]}" --- "${size_changes_params_b[@]}" -- \
-        "${show_changes_env[@]}" --- "${show_changes_params_overrides[@]}" -- \
+        "${version_description}" '-' "${fbs_repo}" \
+        "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
+        "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
+        "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
         "${oemids[@]}"
 )
 # --
 
+# Gets a git tag that can be passed to
+# prepare_env_vars_and_params_for_release.
+#
+# 1 - scripts repo
+# 2 - name of a variable to store the result in
+function git_tag_for_release() {
+    local scripts_repo
+    scripts_repo=${1}; shift
+    local -n git_tag_ref="${1}"; shift
+
+    git_tag_ref=$(cd "${scripts_repo}"; source sdk_lib/sdk_container_common.sh; get_git_version)
+}
+
+# Gets a git tag of a previous nightly that can be passed to
+# prepare_env_vars_and_params_for_bincache.
+#
+# 1 - scripts repo
+# 2 - name of a variable to store the result in
+function git_tag_for_nightly() {
+    local scripts_repo
+    scripts_repo=${1}; shift
+    local -n git_tag_ref="${1}"; shift
+
+    local head_tag search_object
+    head_tag=$(git -C "${scripts_repo}" tag --points-at HEAD)
+    search_object='HEAD'
+    if [[ ${head_tag} = *-nightly-* ]] && [[ ! ${head_tag} = *-INTERMEDIATE ]]; then
+        # HEAD is a nightly, pick an earlier commit to avoid comparing with itself
+        search_object='HEAD^'
+    fi
+    git_tag_ref=$(git -C "${scripts_repo}" describe --tags --abbrev=0 --match='*-nightly-*' --exclude='*-INTERMEDIATE' "${search_object}")
+}
+
+# Gets a list of OEMs that are using sysexts.
+#
+# 1 - scripts repo
+# 2 - name of an array variable to store the result in
 function get_oem_id_list() {
     local scripts_repo
     scripts_repo=${1}; shift
@@ -128,98 +185,36 @@ function get_oem_id_list() {
     done <"${ebuilds[0]}"
 }
 
-# 1 - arch
-# 2 - channel (alpha, beta, stable or lts)
-# 3 - version (FLATCAR_VERSION)
-# 4 - report file (can be relative)
-# 5 - flatcar-build-scripts directory (can be relative, will be realpathed)
+# Generates reports with passed parameters. The report is redirected
+# into the passed report file.
+#
+# 1 - version description (a free form string that describes a version of image that current version is compared against)
+# 2 - report file (can be relative)
+# 3 - flatcar-build-scripts directory (can be relative, will be realpathed)
 # @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides -- list of OEM ids
 #
 # Example:
 #
 # generate_image_changes_report \\
-#     amd64 alpha 3456.0.0+my-changes reports/images.txt ../flatcar-build-scripts .. \\
-#     FROM_B=bincache BOARD_B=amd64-usr --- 3456.0.0+my-changes -- \\
-#     --- bincache:amd64:3456.0.0+my-changes -- \\
-#     "PATH=${PATH}:${PWD}/ci-automation/python-bin" --- -- azure vmware
+#     'Alpha 3456.0.0' reports/images.txt ../flatcar-build-scripts .. \\
+#     FROM_A=release BOARD_A=amd64-usr CHANNEL_A=alpha FROM_B=bincache BOARD_B=amd64-usr --- \\
+#     3456.0.0 3478.0.0+my-changes -- \\
+#     --- \\
+#     release:amd64-usr:3456.0.0 bincache:amd64:3478.0.0+my-changes -- \\
+#     "PATH=${PATH}:${PWD}/ci-automation/python-bin" --- \\
+#     NEW_VERSION=main-3478.0.0-my-changes NEW_CHANNEL=alpha NEW_CHANNEL_PREV_VERSION=3456.0.0 OLD_CHANNEL=alpha OLD_VERSION='' -- \\
+#     azure vmware
 function generate_image_changes_report() (
     set -euo pipefail
 
-    local arch=${1}; shift
-    local channel=${1}; shift
-    local vernum=${1}; shift
+    local version_description=${1}; shift
     local report_output=${1}; shift
     local flatcar_build_scripts_repo=${1}; shift
-
-    local -a package_diff_env package_diff_params
-    local -a size_changes_env size_changes_params
-    local -a show_changes_env show_changes_params
-    local -a oemids
-    local params_shift=0
-
-    split_to_env_and_params \
-        package_diff_env package_diff_params params_shift \
-        "${@}"
-    shift "${params_shift}"
-    split_to_env_and_params \
-        size_changes_env size_changes_params params_shift \
-        "${@}"
-    shift "${params_shift}"
-    split_to_env_and_params \
-        show_changes_env show_changes_params params_shift \
-        "${@}"
-    shift "${params_shift}"
-    oemids=( "${@}" )
-
-    local new_channel new_channel_prev_version channel_a version_a
-    local board="${arch}-usr"
-
-    new_channel="${channel}"
-    new_channel_prev_version=$(channel_version "${new_channel}" "${board}")
-    channel_a=''
-    version_a=''
-    get_channel_a_and_version_a "${new_channel}" "${new_channel_prev_version}" "${vernum}" "${board}" channel_a version_a
-    package_diff_env=(
-        # For A.
-        "FROM_A=release"
-        "BOARD_A=${board}"
-        "CHANNEL_A=${channel_a}"
-        # For B.
-        "${package_diff_env[@]}"
-    )
-    package_diff_params=(
-        # For A.
-        "${version_a}"
-        # For B.
-        "${package_diff_params[@]}"
-    )
-
-    # Nothing to prepend to size_changes_env.
-    #
-    # First parts of the size-changes-report specs, the kind is
-    # appended at call sites.
-    size_changes_params=(
-        # For A.
-        "release:${channel_a}:${board}:${version_a}"
-        # For B.
-        "${size_changes_params[@]}"
-    )
-
-    # Nothing to prepend to show_changes_env.
-    show_changes_params=(
-        "NEW_CHANNEL=${new_channel}"
-        "NEW_CHANNEL_PREV_VERSION=${new_channel_prev_version}"
-        # Potential overrides.
-        "${show_changes_params[@]}"
-    )
+    # rest is forwarded verbatim to print_image_reports
 
     local print_image_reports_invocation=(
         print_image_reports
-        "${flatcar_build_scripts_repo}" "${channel_a}" "${version_a}"
-        "${package_diff_env[@]}" --- "${package_diff_params[@]}" --
-        "${size_changes_env[@]}" --- "${size_changes_params[@]}" --
-        "${show_changes_env[@]}" --- "${show_changes_params[@]}" --
-        "${oemids[@]}"
+        "${flatcar_build_scripts_repo}" "${version_description}" "${@}"
     )
     # Using "|| :" to avoid failing the job.
     if [[ ${report_output} = '-' ]]; then
@@ -232,6 +227,156 @@ function generate_image_changes_report() (
 )
 # --
 
+# Prepares the tool parameters, so they compare against the last
+# release relative to the git tag. The git tag should be in form of
+# <channel>-<version id>-<build id>, which is the usual format used in
+# scripts repo.
+function prepare_env_vars_and_params_for_release() {
+    local arch git_tag
+    arch=${1}; shift
+    git_tag=${1}; shift
+    local -n package_diff_env_ref="${1}"; shift
+    local -n package_diff_params_ref="${1}"; shift
+    local -n size_changes_env_ref="${1}"; shift
+    local -n size_changes_params_ref="${1}"; shift
+    local -n show_changes_env_ref="${1}"; shift
+    local -n show_changes_params_ref="${1}"; shift
+    local -n version_description_ref="${1}"; shift
+
+    local ppfr_channel ppfr_version_id ppfr_build_id ppfr_version ppfr_vernum
+    split_tag "${git_tag}" ppfr_channel ppfr_version_id ppfr_build_id ppfr_version ppfr_vernum
+    if [[ ${ppfr_channel} = 'main' ]]; then
+        ppfr_channel='alpha'
+    fi
+    local board new_channel new_channel_prev_version channel_a version_a
+    board="${arch}-usr"
+
+    new_channel="${ppfr_channel}"
+    new_channel_prev_version=$(channel_version "${new_channel}" "${board}")
+    channel_a=''
+    version_a=''
+    get_channel_a_and_version_a "${new_channel}" "${new_channel_prev_version}" "${ppfr_version}" "${board}" channel_a version_a
+    package_diff_env_ref=(
+        # For A.
+        "FROM_A=release"
+        "BOARD_A=${board}"
+        "CHANNEL_A=${channel_a}"
+    )
+    package_diff_params_ref=(
+        # For A.
+        "${version_a}"
+    )
+
+    # Nothing to prepend to size_changes_env.
+    size_changes_env_ref=()
+    # First parts of the size-changes-report specs, the kind is
+    # appended at call sites.
+    size_changes_params_ref=(
+        # For A.
+        "release:${channel_a}:${board}:${version_a}"
+    )
+
+    # Nothing to prepend to show_changes_env.
+    show_changes_env_ref=()
+    show_changes_params=(
+        "NEW_CHANNEL=${new_channel}"
+        "NEW_CHANNEL_PREV_VERSION=${new_channel_prev_version}"
+        # Channel transition stuff
+        "OLD_CHANNEL=${channel_a}"
+        "OLD_VERSION=${version_a}"
+    )
+
+    version_description_ref="${channel_a} ${version_a}"
+}
+# --
+
+# Prepares the tool parameters, so they compare against the last
+# nightly relative to the git tag. The git tag should be in form of
+# <channel>-<version id>-<build id>, which is the usual format used in
+# scripts repo.
+function prepare_env_vars_and_params_for_bincache() {
+    local arch git_tag
+    arch=${1}; shift
+    git_tag=${1}; shift
+    local -n package_diff_env_ref="${1}"; shift
+    local -n package_diff_params_ref="${1}"; shift
+    local -n size_changes_env_ref="${1}"; shift
+    local -n size_changes_params_ref="${1}"; shift
+    local -n show_changes_env_ref="${1}"; shift
+    local -n show_changes_params_ref="${1}"; shift
+    local -n version_description_ref="${1}"; shift
+
+    local board
+    board="${arch}-usr"
+    local ppfb_channel ppfb_version_id ppfb_build_id ppfb_version ppfb_vernum
+    split_tag "${git_tag}" ppfb_channel ppfb_version_id ppfb_build_id ppfb_version ppfb_vernum
+
+    package_diff_env_ref=(
+        # For A.
+        "FROM_A=bincache"
+        "BOARD_A=${board}"
+        # CHANNEL_A is unused.
+    )
+    package_diff_params_ref=(
+        # For A.
+        "${ppfb_version}"
+    )
+
+    # Nothing to prepend to size_changes_env.
+    size_changes_env_ref=()
+    # First parts of the size-changes-report specs, the kind is
+    # appended at call sites.
+    size_changes_params_ref=(
+        # For A.
+        "bincache:${arch}:${ppfb_version}"
+    )
+
+    # Nothing to prepend to show_changes_env.
+    show_changes_env_ref=()
+    show_changes_params=(
+        "NEW_CHANNEL=${ppfb_channel}"
+        "NEW_CHANNEL_PREV_VERSION=${ppfb_vernum}"
+        # Channel transition stuff, we set the old channel to be the
+        # same as the new channel to say that there was no channel
+        # transition. Such would not make any sense here.
+        "OLD_CHANNEL=${ppfb_channel}"
+        "OLD_VERSION=${ppfb_vernum}"
+    )
+
+    version_description_ref="development version ${ppfb_channel} ${ppfb_version}"
+}
+# --
+
+function split_tag() {
+    local git_tag
+    git_tag=${1}; shift
+    local -n channel_ref=${1}; shift
+    local -n version_id_ref=${1}; shift
+    local -n build_id_ref=${1}; shift
+    local -n version_ref=${1}; shift
+    local -n vernum_ref=${1}; shift
+
+    local channel version_id build_id version vernum
+    channel=${git_tag%%-*}
+    version_id=${git_tag#*-}
+    version_id=${version_id%%-*}
+    build_id=${git_tag#"${channel}-${version_id}"}
+    if [[ -n ${build_id} ]]; then
+        build_id=${build_id#-}
+        version="${version_id}+${build_id}"
+        vernum="${version_id}-${build_id}"
+    else
+        version="${version_id}"
+        vernum="${version_id}"
+    fi
+    channel_ref=${channel}
+    version_id_ref=${version_id}
+    build_id_ref=${build_id}
+    version_ref=${version}
+    vernum_ref=${vernum}
+}
+# --
+
 function get_channel_a_and_version_a() {
     local new_channel=${1}; shift
     local new_channel_prev_version=${1}; shift
@@ -290,7 +435,7 @@ function channel_version() {
 # flatcar-build-scripts repo. The environment and parameters for the
 # scripts are passed as follows:
 #
-# print_image_reports <flatcar-build-scripts-directory> <channel a> <version a> \\
+# print_image_reports <flatcar-build-scripts-directory> <previous version description> \\
 #       <env vars for package-diff> --- <parameters for package-diff> -- \\
 #       <env vars for size-change-report.sh> --- <parameters for size-change-report.sh> -- \\
 #       <env vars for show-changes> --- <parameters for show-changes> -- \\
@@ -314,11 +459,11 @@ function channel_version() {
 #
 # Should come in format of key=value, just like env vars. It's
 # expected that the following key-value pairs will be specified - for
-# NEW_CHANNEL, NEW_CHANNEL_PREV_VERSION NEW_VERSION.
+# NEW_CHANNEL, NEW_CHANNEL_PREV_VERSION, NEW_VERSION, OLD_CHANNEL and
+# OLD_VERSION.
 function print_image_reports() {
     local flatcar_build_scripts_repo=${1}; shift
-    local channel_a=${1}; shift
-    local version_a=${1}; shift
+    local previous_version_description=${1}; shift
     local -a package_diff_env=() package_diff_params=()
     local -a size_change_report_env=() size_change_report_params=()
     local -a show_changes_env=() show_changes_params=()
@@ -349,40 +494,40 @@ function print_image_reports() {
         "${flatcar_build_scripts_repo}/size-change-report.sh"
     )
 
-    echo "== Image differences compared to ${channel_a} ${version_a} =="
-    echo "Package updates, compared to ${channel_a} ${version_a}:"
+    echo "== Image differences compared to ${previous_version_description} =="
+    echo "Package updates, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_packages.txt \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
-    echo "Image file changes, compared to ${channel_a} ${version_a}:"
+    echo "Image file changes, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
-    echo "Image file size changes, compared to ${channel_a} ${version_a}:"
+    echo "Image file size changes, compared to ${previous_version_description}:"
     if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:wtd}" 2>&1; then
         "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:old}" 2>&1
     fi
     echo
-    echo "Image kernel config changes, compared to ${channel_a} ${version_a}:"
+    echo "Image kernel config changes, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_kernel_config.txt \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
-    echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${channel_a} ${version_a}:"
+    echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt CALCSIZE=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
 
-    echo "== Init ramdisk differences compared to ${channel_a} ${version_a} =="
-    echo "Image init ramdisk file changes, compared to ${channel_a} ${version_a}:"
+    echo "== Init ramdisk differences compared to ${previous_version_description} =="
+    echo "Image init ramdisk file changes, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_initrd_contents.txt FILESONLY=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
     echo
-    echo "Image init ramdisk file size changes, compared to ${channel_a} ${version_a}:"
+    echo "Image init ramdisk file size changes, compared to ${previous_version_description}:"
     if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-wtd}" 2>&1; then
         "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-old}" 2>&1
     fi
@@ -393,18 +538,18 @@ function print_image_reports() {
 
     local oemid
     for oemid in "${oemids[@]}"; do
-        echo "== Sysext changes for OEM ${oemid} compared to ${channel_a} ${version_a} =="
-        echo "Package updates, compared to ${channel_a} ${version_a}:"
+        echo "== Sysext changes for OEM ${oemid} compared to ${previous_version_description} =="
+        echo "Package updates, compared to ${previous_version_description}:"
         env \
             "${package_diff_env[@]}" FILE="oem-${oemid}_packages.txt" \
             "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
         echo
-        echo "Image file changes, compared to ${channel_a} ${version_a}:"
+        echo "Image file changes, compared to ${previous_version_description}:"
         env \
             "${package_diff_env[@]}" FILE="oem-${oemid}_contents.txt" FILESONLY=1 CUTKERNEL=1 \
             "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
         echo
-        echo "Image file size changes, compared to ${channel_a} ${version_a}:"
+        echo "Image file size changes, compared to ${previous_version_description}:"
         if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-wtd}"; then
             "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-old}" 2>&1
         fi
@@ -415,18 +560,22 @@ function print_image_reports() {
     for param in "${show_changes_params[@]}"; do
         local "SHOW_CHANGES_${param}"
     done
-    # The first changelog we print is always against the previous version of the new channel (is only same as ${channel_a} ${version_a} without a transition)
+    # The first changelog we print is always against the previous
+    # version of the new channel (is only same as old channel and old
+    # version without a transition)
     env \
         "${show_changes_env[@]}" \
         "${flatcar_build_scripts_repo}/show-changes" \
         "${SHOW_CHANGES_NEW_CHANNEL}-${SHOW_CHANGES_NEW_CHANNEL_PREV_VERSION}" \
         "${SHOW_CHANGES_NEW_VERSION}" 2>&1
-    # See if a channel transition happened and print the changelog against ${channel_a} ${version_a} which is the previous release
-    if [ "${channel_a}" != "${SHOW_CHANGES_NEW_CHANNEL}" ]; then
+    # See if a channel transition happened and print the changelog
+    # against old channel and old version which is the previous
+    # release
+    if [ "${SHOW_CHANGES_OLD_CHANNEL}" != "${SHOW_CHANGES_NEW_CHANNEL}" ]; then
         env \
             "${show_changes_env[@]}" \
             "${flatcar_build_scripts_repo}/show-changes" \
-            "${channel_a}-${version_a}" \
+            "${SHOW_CHANGES_OLD_CHANNEL}-${SHOW_CHANGES_OLD_VERSION}" \
             "${SHOW_CHANGES_NEW_VERSION}" 2>&1
     fi
 }

From 2c7af82d75a87646c5c7d944a4f1446c8d0ebf70 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 21 Sep 2023 16:51:43 +0200
Subject: [PATCH 06/32] ci-automation/image-changes: Shrink a clone of
 flatcar-build-script

---
 ci-automation/image_changes.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 13691d6860..6fc25c12a6 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -97,6 +97,7 @@ function image_changes() (
     rm -rf "${fbs_repo}"
     git clone \
         --depth 1 \
+        --single-branch \
         "https://github.com/flatcar/flatcar-build-scripts" \
         "${fbs_repo}"
     if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then

From 50b515ee3250b3a2d36480924f68d99967d6b1df Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Fri, 22 Sep 2023 10:06:50 +0200
Subject: [PATCH 07/32] ci-automation/image-changes: Expect HEAD commit to have
 a tag

---
 ci-automation/image_changes.sh | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 6fc25c12a6..930e75b63e 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -65,7 +65,8 @@ function image_changes() (
             ;;
     esac
 
-    local version
+    local ic_head_tag version
+    head_git_tag . ic_head_tag
     version=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
     package_diff_env+=(
         "FROM_B=bincache"
@@ -90,7 +91,7 @@ function image_changes() (
     show_changes_params+=(
         # The show-changes script expects a tag name, so using git tag
         # here instead of the vernum variable.
-        "NEW_VERSION=$(git tag --points-at HEAD)"
+        "NEW_VERSION=${ic_head_tag}"
     )
 
     local fbs_repo='../flatcar-build-scripts'
@@ -122,11 +123,23 @@ function image_changes() (
 # 1 - scripts repo
 # 2 - name of a variable to store the result in
 function git_tag_for_release() {
+    local scripts_repo git_tag_var_name
+    scripts_repo=${1}; shift
+    git_tag_var_name=${1}; shift
+
+    head_git_tag "${scripts_repo}" "${git_tag_var_name}"
+}
+
+function head_git_tag() {
     local scripts_repo
     scripts_repo=${1}; shift
     local -n git_tag_ref="${1}"; shift
 
-    git_tag_ref=$(cd "${scripts_repo}"; source sdk_lib/sdk_container_common.sh; get_git_version)
+    git_tag_ref=$(git -C "${scripts_repo}" tag --points-at HEAD)
+    if [[ -z ${git_tag_ref} ]]; then
+        echo 'expected git HEAD commit to contain a tag' >&2
+        exit 1
+    fi
 }
 
 # Gets a git tag of a previous nightly that can be passed to

From 123ae2471fe2e8856934c333fa5b80858a73de3b Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Fri, 22 Sep 2023 11:37:51 +0200
Subject: [PATCH 08/32] ci-automation/image-changes: Make messages stand out

---
 ci-automation/image_changes.sh | 95 +++++++++++++++++++++++++---------
 1 file changed, 71 insertions(+), 24 deletions(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 930e75b63e..149c3526c7 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -500,51 +500,49 @@ function print_image_reports() {
 
     flatcar_build_scripts_repo=$(realpath "${flatcar_build_scripts_repo}")
 
-    echo "==================================================================="
-
     local size_changes_invocation=(
         env
         "${size_change_report_env[@]}"
         "${flatcar_build_scripts_repo}/size-change-report.sh"
     )
 
-    echo "== Image differences compared to ${previous_version_description} =="
-    echo "Package updates, compared to ${previous_version_description}:"
+    yell "Image differences compared to ${previous_version_description}"
+    underline "Package updates, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_packages.txt \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-    echo
-    echo "Image file changes, compared to ${previous_version_description}:"
+
+    underline "Image file changes, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-    echo
-    echo "Image file size changes, compared to ${previous_version_description}:"
+
+    underline "Image file size changes, compared to ${previous_version_description}:"
     if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:wtd}" 2>&1; then
         "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:old}" 2>&1
     fi
-    echo
-    echo "Image kernel config changes, compared to ${previous_version_description}:"
+
+    underline "Image kernel config changes, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_kernel_config.txt \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-    echo
-    echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${previous_version_description}:"
+
+    underline "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt CALCSIZE=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-    echo
 
-    echo "== Init ramdisk differences compared to ${previous_version_description} =="
-    echo "Image init ramdisk file changes, compared to ${previous_version_description}:"
+    yell "Init ramdisk differences compared to ${previous_version_description}"
+    underline "Image init ramdisk file changes, compared to ${previous_version_description}:"
     env \
         "${package_diff_env[@]}" FILE=flatcar_production_image_initrd_contents.txt FILESONLY=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-    echo
-    echo "Image init ramdisk file size changes, compared to ${previous_version_description}:"
+
+    underline "Image init ramdisk file size changes, compared to ${previous_version_description}:"
     if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-wtd}" 2>&1; then
         "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-old}" 2>&1
     fi
+    echo
     echo "Take the total size difference with a grain of salt as normally initrd is compressed, so the actual difference will be smaller."
     echo "To see the actual difference in size, see if there was a report for /boot/flatcar/vmlinuz-a."
     echo "Note that vmlinuz-a also contains the kernel code, which might have changed too, so the reported difference does not accurately describe the change in initrd."
@@ -552,22 +550,21 @@ function print_image_reports() {
 
     local oemid
     for oemid in "${oemids[@]}"; do
-        echo "== Sysext changes for OEM ${oemid} compared to ${previous_version_description} =="
-        echo "Package updates, compared to ${previous_version_description}:"
+        yell "Sysext changes for OEM ${oemid} compared to ${previous_version_description}"
+        underline "Package updates, compared to ${previous_version_description}:"
         env \
             "${package_diff_env[@]}" FILE="oem-${oemid}_packages.txt" \
             "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-        echo
-        echo "Image file changes, compared to ${previous_version_description}:"
+
+        underline "Image file changes, compared to ${previous_version_description}:"
         env \
             "${package_diff_env[@]}" FILE="oem-${oemid}_contents.txt" FILESONLY=1 CUTKERNEL=1 \
             "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
-        echo
-        echo "Image file size changes, compared to ${previous_version_description}:"
+
+        underline "Image file size changes, compared to ${previous_version_description}:"
         if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-wtd}"; then
             "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-old}" 2>&1
         fi
-        echo
     done
 
     local param
@@ -577,6 +574,7 @@ function print_image_reports() {
     # The first changelog we print is always against the previous
     # version of the new channel (is only same as old channel and old
     # version without a transition)
+    yell "Changelog against ${SHOW_CHANGES_NEW_CHANNEL}-${SHOW_CHANGES_NEW_CHANNEL_PREV_VERSION}"
     env \
         "${show_changes_env[@]}" \
         "${flatcar_build_scripts_repo}/show-changes" \
@@ -586,6 +584,7 @@ function print_image_reports() {
     # against old channel and old version which is the previous
     # release
     if [ "${SHOW_CHANGES_OLD_CHANNEL}" != "${SHOW_CHANGES_NEW_CHANNEL}" ]; then
+        yell "Changelog against ${SHOW_CHANGES_OLD_CHANNEL}-${SHOW_CHANGES_OLD_VERSION}"
         env \
             "${show_changes_env[@]}" \
             "${flatcar_build_scripts_repo}/show-changes" \
@@ -595,6 +594,54 @@ function print_image_reports() {
 }
 # --
 
+function yell() {
+    local msg
+    msg=${1}; shift
+
+    local msg_len
+    msg_len=${#msg}
+
+    local y_str
+    repeat_string '!' $((msg_len + 6)) y_str
+
+    printf '\n%s\n!! %s !!\n%s\n\n' "${y_str}" "${msg}" "${y_str}"
+}
+
+function underline() {
+    local msg
+    msg=${1}; shift
+
+    local msg_len
+    msg_len=${#msg}
+
+    local u_str
+    repeat_string '=' "${msg_len}" u_str
+
+    printf '\n%s\n%s\n\n' "${msg}" "${u_str}"
+}
+
+function repeat_string() {
+    local str ntimes out_str_var_name
+    str="${1}"; shift
+    ntimes="${1}"; shift
+    out_str_var_name="${1}"; shift
+    local -n out_str_ref="${out_str_var_name}"
+
+    if [[ ${ntimes} -eq 0 ]]; then
+        out_str_ref=""
+        return 0
+    elif [[ ${ntimes} -eq 1 ]]; then
+        out_str_ref="${str}"
+        return 0
+    fi
+    local add_one
+    add_one=$((ntimes % 2))
+    repeat_string "${str}${str}" $((ntimes / 2)) "${out_str_var_name}"
+    if [[ add_one -gt 0 ]]; then
+        out_str_ref+="${str}"
+    fi
+}
+
 # 1 - name of an array variable for environment variables
 # 2 - name of an array variable for parameters
 # 3 - name of a scalar variable for shift number

From cda692ad3c5ca827d823671c2ea249d7557e470b Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Fri, 22 Sep 2023 16:14:42 +0200
Subject: [PATCH 09/32] ci-automation/image-changes: Strip kernel versions in
 initrd reports

This is something that we already do for production image and for OEM
sysext images.
---
 ci-automation/image_changes.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 149c3526c7..554bd1122a 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -535,7 +535,7 @@ function print_image_reports() {
     yell "Init ramdisk differences compared to ${previous_version_description}"
     underline "Image init ramdisk file changes, compared to ${previous_version_description}:"
     env \
-        "${package_diff_env[@]}" FILE=flatcar_production_image_initrd_contents.txt FILESONLY=1 \
+        "${package_diff_env[@]}" FILE=flatcar_production_image_initrd_contents.txt FILESONLY=1 CUTKERNEL=1 \
         "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
 
     underline "Image init ramdisk file size changes, compared to ${previous_version_description}:"

From 170e17a67390c09871de3f05866ecac8ede8b23d Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 28 Sep 2023 08:02:25 +0200
Subject: [PATCH 10/32] ci-automation/image-changes: Second parameter is not
 optional

---
 ci-automation/image_changes.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 554bd1122a..7303a000bf 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -19,7 +19,7 @@
 # INPUT:
 #
 #   1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
-#   2. What to compare against, can be "last-release" or "last-nightly".
+#   2. What to compare against, must be either "last-release" or "last-nightly".
 #
 # OPTIONAL INPUT:
 #
@@ -33,9 +33,10 @@
 # variables are not spilled into the caller.
 function image_changes() (
     set -euo pipefail
+    local arch what
 
-    local arch=${1}; shift
-    local what=${1-last-release}; shift
+    arch=${1}; shift
+    what=${1}; shift
 
     local -a package_diff_env package_diff_params
     local -a size_changes_env size_changes_params

From 769801643915d0d9cc8c960a9a6bb210f0746363 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Fri, 29 Sep 2023 14:35:35 +0200
Subject: [PATCH 11/32] ci-automation/image-changes: Document - as standard
 output

---
 ci-automation/image_changes.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 7303a000bf..fe8ad0dd6e 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -204,7 +204,7 @@ function get_oem_id_list() {
 # into the passed report file.
 #
 # 1 - version description (a free form string that describes a version of image that current version is compared against)
-# 2 - report file (can be relative)
+# 2 - report file (can be relative), '-' for standard output
 # 3 - flatcar-build-scripts directory (can be relative, will be realpathed)
 # @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides -- list of OEM ids
 #

From 8b5e6f0f2ce9de2162acf11c095d6d74af744a75 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 28 Sep 2023 18:02:06 +0200
Subject: [PATCH 12/32] ci-automation/image-changes: Do not expect HEAD commit
 to have a tag

It certainly does happen on GitHub Actions that the HEAD commit has no
tag. In such case, fake the tag from version file. The git tag in this
scenario is used to figure out a channel transition.
---
 ci-automation/image_changes.sh | 39 ++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index fe8ad0dd6e..7734d5b361 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -129,6 +129,42 @@ function git_tag_for_release() {
     git_tag_var_name=${1}; shift
 
     head_git_tag "${scripts_repo}" "${git_tag_var_name}"
+
+    local -n git_tag_ref="${git_tag_var_name}"
+    local version_file version_id build_id minor_version channel
+    if [[ ${git_tag_ref} = 'HEAD' ]]; then
+        # Welp, we wanted to have something in form of
+        # <channel>-<version_id>-<build_id>, fake something up from
+        # version file. Figuring out the channel is a heuristic at
+        # best.
+        version_file="${scripts_repo}/sdk_container/.repo/manifests/version.txt"
+        if [[ ! -e ${version_file} ]]; then
+            echo "The scripts repo at '${scripts_repo}' is messed up, has no version file" >&2
+            exit 1
+        fi
+        version_id=$(source "${version_file}"; printf '%s' "${FLATCAR_VERSION_ID}")
+        build_id=$(source "${version_file}"; printf '%s' "${FLATCAR_BUILD_ID}")
+        minor_version=${version_id#*.}
+        minor_version=${minor_version%.*}
+        case ${minor_version} in
+            0)
+                channel=alpha
+                ;;
+            1)
+                channel=beta
+                ;;
+            2)
+                channel=stable
+                ;;
+            3)
+                channel=lts
+                ;;
+            *)
+                channel=main
+                ;;
+        esac
+        git_tag_ref="${channel}-${version_id}-${build_id}"
+    fi
 }
 
 function head_git_tag() {
@@ -138,8 +174,7 @@ function head_git_tag() {
 
     git_tag_ref=$(git -C "${scripts_repo}" tag --points-at HEAD)
     if [[ -z ${git_tag_ref} ]]; then
-        echo 'expected git HEAD commit to contain a tag' >&2
-        exit 1
+        git_tag_ref='HEAD'
     fi
 }
 

From 202b905a47e87b4511e9aed5f34076073cdaf8a9 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Fri, 29 Sep 2023 07:34:55 +0200
Subject: [PATCH 13/32] .github/ci: Fix clobbering of parameters and env vars

---
 .github/workflows/ci.yaml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7826195c24..a635d92693 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -222,12 +222,13 @@ jobs:
           size_changes_params+=(
               "local:${PWD}/artifacts/${arch}-usr/latest"
           )
-          show_changes_env=(
+          show_changes_env+=(
+              # Override the default locations of repositories.
               "SCRIPTS_REPO=."
               "COREOS_OVERLAY_REPO=../coreos-overlay"
               "PORTAGE_STABLE_REPO=../portage-stable"
           )
-          show_changes_params=(
+          show_changes_params+=(
               # We may not have a tag handy, so we tell show-changes
               # to use git HEAD as a reference to new changelog
               # entries.
@@ -278,12 +279,13 @@ jobs:
           size_changes_params+=(
               "local:${PWD}/artifacts/${arch}-usr/latest"
           )
-          show_changes_env=(
+          show_changes_env+=(
+              # Override the default locations of repositories.
               "SCRIPTS_REPO=."
               "COREOS_OVERLAY_REPO=../coreos-overlay"
               "PORTAGE_STABLE_REPO=../portage-stable"
           )
-          show_changes_params=(
+          show_changes_params+=(
               # We may not have a tag handy, so we tell show-changes
               # to use git HEAD as a reference to new changelog
               # entries.

From c03d372c548e8025e28c3e2853221dfbb2784137 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Fri, 29 Sep 2023 14:37:36 +0200
Subject: [PATCH 14/32] .github/ci: Run image reports job after vms job

It shows changes also in OEM images and these are built in vms step.
---
 .github/workflows/ci.yaml | 92 +++++++++++++++++++--------------------
 1 file changed, 46 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index a635d92693..776c786069 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -187,6 +187,52 @@ jobs:
                                 --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \
                                 prodtar container
 
+      - name: Build VM image(s)
+        shell: bash
+        run: |
+          set -euo pipefail
+          set -x
+
+          source ci-automation/ci_automation_common.sh
+
+          images_out="images"
+
+          has_packet=0
+          has_pxe=0
+          formats="${IMAGE_FORMATS}"
+          for format in "${formats}";do
+            [[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1
+            [[ "${format}" = 'pxe' ]] && has_pxe=1
+          done
+
+          [[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}"
+          if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then
+            formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d')
+            formats+=" vmware vmware_insecure vmware_ova vmware_raw"
+          fi
+          if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then
+            formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d')
+            formats+=" ami ami_vmdk"
+          fi
+          # Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
+          formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
+
+          for format in ${formats}; do
+              echo " ###################  VENDOR '${format}' ################### "
+              ./run_sdk_container -n "${container_name}" \
+                  ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
+                      --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
+                      --image_compression_formats=bz2
+          done
+
+          # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
+          #  so we move things around.
+          mkdir -p artifacts/images
+          (
+            cd artifacts/${arch}-usr/latest/
+            mv * ../../images/
+          )
+
       - name: Generate reports against last release
         shell: bash
         run: |
@@ -301,52 +347,6 @@ jobs:
               "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
               "${oemids[@]}"
 
-      - name: Build VM image(s)
-        shell: bash
-        run: |
-          set -euo pipefail
-          set -x
-
-          source ci-automation/ci_automation_common.sh
-
-          images_out="images"
-
-          has_packet=0
-          has_pxe=0
-          formats="${IMAGE_FORMATS}"
-          for format in "${formats}";do
-            [[ "${format}" = 'packet' ]] || [[ "${format}" = 'equinix_metal' ]] && has_packet=1
-            [[ "${format}" = 'pxe' ]] && has_pxe=1
-          done
-
-          [[ ${has_packet} -eq 1 ]] && [[ ${has_pxe} -eq 0 ]] && set -- 'pxe' "${@}"
-          if echo "$formats" | tr ' ' '\n' | grep -q '^vmware'; then
-            formats=$(echo "$formats" | tr ' ' '\n' | sed '/vmware.*/d')
-            formats+=" vmware vmware_insecure vmware_ova vmware_raw"
-          fi
-          if echo "$formats" | tr ' ' '\n' | grep -q -P '^(ami|aws)'; then
-            formats=$(echo "$formats" | tr ' ' '\n' | sed '/ami.*/d' | sed '/aws/d')
-            formats+=" ami ami_vmdk"
-          fi
-          # Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
-          formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
-
-          for format in ${formats}; do
-              echo " ###################  VENDOR '${format}' ################### "
-              ./run_sdk_container -n "${container_name}" \
-                  ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
-                      --from "${CI_CONTAINER_ARTIFACT_ROOT}/${arch}-usr/latest" \
-                      --image_compression_formats=bz2
-          done
-
-          # upload-artifacts cannot handle artifact uploads from sym-linked directories (no, really)
-          #  so we move things around.
-          mkdir -p artifacts/images
-          (
-            cd artifacts/${arch}-usr/latest/
-            mv * ../../images/
-          )
-
       - name: Upload binpkgs
         uses: actions/upload-artifact@v3
         with:

From a7853de1744f01395b13db24d2e7358e844b1a98 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 12 Oct 2023 12:28:37 +0200
Subject: [PATCH 15/32] overlay coreos-base/common-oem-files: Add some arch
 info about OEM

For now we don't have a single place where we could learn about which
OEMs with a sysext image gets built for certain architecture. So add
it for now to the coreos-base/common-oem-files package. Missing
architecture info for an OEM that gets built only in a certain arch is
not going to be fatal for the image changes job - we will just get a
report for this OEM with messages about failures to download some
files.
---
 .../common-oem-files/common-oem-files-0-r4.ebuild      | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
index 108c67b88d..16b385bac9 100644
--- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
+++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
@@ -3,14 +3,20 @@
 
 EAPI=8
 
+# One OEM ID per line, a comment at the end of the line to denote
+# which arch this OEM is for (not necessary if OEM is built for all of
+# them). The arches should be space separated.
+#
+# This is used by the ci-automation/image_changes.sh script to figure
+# out the per-arch OEM IDs.
 OEMIDS=(
     ami
     azure
-    digitalocean
+    digitalocean # amd64
     openstack
     packet
     qemu
-    vmware
+    vmware # amd64
 )
 
 DESCRIPTION='Common OEM files'

From d373052ca4bb4b4d0c028c59cee6c99dede73f8f Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 12 Oct 2023 12:32:43 +0200
Subject: [PATCH 16/32] ci-automation/image-changes: Filter out OEM IDs not
 built for an arch

This is to limit the amount of reports consisting purely of failures,
because some files were missing. And those files will be missing,
because an OEM might not even have any image for certain arches (like
digitalocean has no arm64 images).
---
 .github/workflows/ci.yaml      |  4 ++--
 ci-automation/image_changes.sh | 42 ++++++++++++++++++++++++++--------
 2 files changed, 35 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 776c786069..1367804dc3 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -282,7 +282,7 @@ jobs:
           )
 
           declare -a oemids
-          get_oem_id_list . oemids
+          get_oem_id_list . "${arch}" oemids
           generate_image_changes_report \
               "${version_description}" 'image-changes-reports-release.txt' "../flatcar-build-scripts" \
               "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
@@ -339,7 +339,7 @@ jobs:
           )
 
           declare -a oemids
-          get_oem_id_list . oemids
+          get_oem_id_list . "${arch}" oemids
           generate_image_changes_report \
               "${version_description}" 'image-changes-reports-nightly.txt' "../flatcar-build-scripts" \
               "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 7734d5b361..e3ed950508 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -108,7 +108,7 @@ function image_changes() (
     echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${version}/flatcar_production_image.bin.bz2"
     echo
     local -a oemids
-    get_oem_id_list . oemids
+    get_oem_id_list . "${arch}" oemids
     generate_image_changes_report \
         "${version_description}" '-' "${fbs_repo}" \
         "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
@@ -201,10 +201,12 @@ function git_tag_for_nightly() {
 # Gets a list of OEMs that are using sysexts.
 #
 # 1 - scripts repo
-# 2 - name of an array variable to store the result in
+# 2 - arch
+# 3 - name of an array variable to store the result in
 function get_oem_id_list() {
-    local scripts_repo
+    local scripts_repo arch
     scripts_repo=${1}; shift
+    arch=${1}; shift
     local -n list_var_ref=${1}; shift
 
     local -a ebuilds
@@ -214,22 +216,44 @@ function get_oem_id_list() {
     if [[ ${#ebuilds[@]} -eq 0 ]]; then
         return 0
     fi
-    local line mode
-    # 0 = none OEMIDS line found yet
+    local mode
+    # 0 = no OEMIDS line found yet
     # 1 = OEMIDS line found
     mode=0
-    while read -r line; do
+    local -a fields
+    local first arch_field arch_found
+    while read -r -a fields; do
+        if [[ ${#fields[@]} -eq 0 ]]; then
+            continue
+        fi
+        first=${fields[0]}
         case ${mode} in
             0)
-                if [[ ${line} = 'OEMIDS=(' ]]; then
+                if [[ ${first} = 'OEMIDS=(' ]]; then
                     mode=1
                 fi
                 ;;
             1)
-                if [[ ${line} = ')' ]]; then
+                if [[ ${first} = ')' ]]; then
                     break
                 fi
-                list_var_ref+=( "${line}" )
+                if [[ ${#fields[@]} -gt 1 ]]; then
+                    if [[ ${fields[1]} != '#' ]]; then
+                        echo "expect a line inside OEMIDS to be like '<OEMID> # <ARCH1> <ARCH2>…' or just '<OEMID>', got '${fields[*]}'" >&2
+                        exit 1
+                    fi
+                    arch_found=
+                    for arch_field in "${fields[@]:2}"; do
+                        if [[ ${arch} = "${arch_field}" ]]; then
+                            arch_found=x
+                            break
+                        fi
+                    done
+                    if [[ -z ${arch_found} ]]; then
+                        continue
+                    fi
+                fi
+                list_var_ref+=( "${first}" )
                 ;;
         esac
     done <"${ebuilds[0]}"

From 5b3e90eceb208029c8df8042a064ce1bf3e78f49 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 12 Oct 2023 17:10:54 +0200
Subject: [PATCH 17/32] .github/ci: Fix artifacts paths

---
 .github/workflows/ci.yaml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1367804dc3..cccfbde61c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -255,7 +255,7 @@ jobs:
           prepare_env_vars_and_params_for_release "${arch}" "${git_tag}" "${var_names[@]}"
 
           package_diff_env+=(
-              "FROM_B=file://${PWD}/artifacts/${arch}-usr/latest"
+              "FROM_B=file://${PWD}/artifacts/images"
               # BOARD_B and CHANNEL_B are unused.
           )
           package_diff_params+=(
@@ -266,7 +266,7 @@ jobs:
           )
           # Nothing to add to size changes env.
           size_changes_params+=(
-              "local:${PWD}/artifacts/${arch}-usr/latest"
+              "local:${PWD}/artifacts/images"
           )
           show_changes_env+=(
               # Override the default locations of repositories.
@@ -312,7 +312,7 @@ jobs:
           prepare_env_vars_and_params_for_bincache "${arch}" "${git_tag}" "${var_names[@]}"
 
           package_diff_env+=(
-              "FROM_B=file://${PWD}/artifacts/${arch}-usr/latest"
+              "FROM_B=file://${PWD}/artifacts/images"
               # BOARD_B and CHANNEL_B are unused.
           )
           package_diff_params+=(
@@ -323,7 +323,7 @@ jobs:
           )
           # Nothing to add to size changes env.
           size_changes_params+=(
-              "local:${PWD}/artifacts/${arch}-usr/latest"
+              "local:${PWD}/artifacts/images"
           )
           show_changes_env+=(
               # Override the default locations of repositories.

From ead79f989cd38b74015f835fdaba6802bcf10771 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 12 Oct 2023 17:11:33 +0200
Subject: [PATCH 18/32] ci-automation/image-changes: Source version.txt

This avoid messing with quotes.
---
 ci-automation/image_changes.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index e3ed950508..8fc574377e 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -488,21 +488,21 @@ function get_channel_a_and_version_a() {
 # --
 
 # Gets the latest release for given channel and board. For lts channel
-# gets a version of the latest LTS.
-function channel_version() {
+# gets a version of the latest LTS. Runs in a subshell.
+function channel_version() (
     local channel=${1}; shift
     local board=${1}; shift
 
-    curl \
-        -fsSL \
-        --retry-delay 1 \
-        --retry 60 \
-        --retry-connrefused \
-        --retry-max-time 60 \
-        --connect-timeout 20 \
-        "https://${channel}.release.flatcar-linux.net/${board}/current/version.txt" | \
-        grep -m 1 'FLATCAR_VERSION=' | cut -d = -f 2-
-}
+    source <(curl \
+                 -fsSL \
+                 --retry-delay 1 \
+                 --retry 60 \
+                 --retry-connrefused \
+                 --retry-max-time 60 \
+                 --connect-timeout 20 \
+                 "https://${channel}.release.flatcar-linux.net/${board}/current/version.txt")
+    echo "${FLATCAR_VERSION}"
+)
 # --
 
 # Prints some reports using scripts from the passed path to

From dabb54783de823119f3dc8d106aa86e6614adfc8 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 26 Oct 2023 13:11:04 +0200
Subject: [PATCH 19/32] .github/ci, ci-automation/image-changes: Print diffs
 for base sysexts

---
 .github/workflows/ci.yaml      |  5 ++-
 ci-automation/image_changes.sh | 76 ++++++++++++++++++++++++++++++----
 2 files changed, 72 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index cccfbde61c..01921d508a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -281,14 +281,15 @@ jobs:
               'NEW_VERSION=HEAD'
           )
 
-          declare -a oemids
+          declare -a oemids base_sysexts
           get_oem_id_list . "${arch}" oemids
+          get_base_sysext_list . base_sysexts
           generate_image_changes_report \
               "${version_description}" 'image-changes-reports-release.txt' "../flatcar-build-scripts" \
               "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
               "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
               "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
-              "${oemids[@]}"
+              "${oemids[@]}" -- "${base_sysexts[@]}"
 
       - name: Generate reports against last nightly
         shell: bash
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 8fc574377e..5848d03e9a 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -107,14 +107,15 @@ function image_changes() (
     fi
     echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${version}/flatcar_production_image.bin.bz2"
     echo
-    local -a oemids
+    local -a oemids base_sysexts
     get_oem_id_list . "${arch}" oemids
+    get_base_sysext_list . base_sysexts
     generate_image_changes_report \
         "${version_description}" '-' "${fbs_repo}" \
         "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
         "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
         "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
-        "${oemids[@]}"
+        "${oemids[@]}" -- "${base_sysexts[@]}"
 )
 # --
 
@@ -259,13 +260,32 @@ function get_oem_id_list() {
     done <"${ebuilds[0]}"
 }
 
+function get_base_sysext_list() {
+    local scripts_repo=${1}; shift
+    local -n list_var_ref=${1}; shift
+
+    local line
+    line=$({ git -C "${scripts_repo}" grep -F 'DEFINE_string base_sysexts ' | head -n1; } || :)
+    line=${line#*'"'}
+    line=${line%'"'*}
+
+    list_var_ref=()
+    local -a entries
+    mapfile -t entries <<<"${line//,/$'\n'}"
+    local entry
+    for entry in "${entries[@]}"; do
+        list_var_ref+=( "${entry%%:*}" )
+    done
+}
+
+
 # Generates reports with passed parameters. The report is redirected
 # into the passed report file.
 #
 # 1 - version description (a free form string that describes a version of image that current version is compared against)
 # 2 - report file (can be relative), '-' for standard output
 # 3 - flatcar-build-scripts directory (can be relative, will be realpathed)
-# @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides -- list of OEM ids
+# @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides -- list of OEM ids -- list of base sysext names
 #
 # Example:
 #
@@ -277,7 +297,7 @@ function get_oem_id_list() {
 #     release:amd64-usr:3456.0.0 bincache:amd64:3478.0.0+my-changes -- \\
 #     "PATH=${PATH}:${PWD}/ci-automation/python-bin" --- \\
 #     NEW_VERSION=main-3478.0.0-my-changes NEW_CHANNEL=alpha NEW_CHANNEL_PREV_VERSION=3456.0.0 OLD_CHANNEL=alpha OLD_VERSION='' -- \\
-#     azure vmware
+#     azure vmware -- containerd-flatcar docker-flatcar
 function generate_image_changes_report() (
     set -euo pipefail
 
@@ -513,7 +533,7 @@ function channel_version() (
 #       <env vars for package-diff> --- <parameters for package-diff> -- \\
 #       <env vars for size-change-report.sh> --- <parameters for size-change-report.sh> -- \\
 #       <env vars for show-changes> --- <parameters for show-changes> -- \\
-#       <list of OEM ids>
+#       <list of OEM ids> -- <list of base sysexts>
 #
 # Env vars are passed to the called scripts verbatim. Parameters are
 # described below.
@@ -541,7 +561,7 @@ function print_image_reports() {
     local -a package_diff_env=() package_diff_params=()
     local -a size_change_report_env=() size_change_report_params=()
     local -a show_changes_env=() show_changes_params=()
-    local -a oemids
+    local -a oemids base_sysexts
     local params_shift=0
 
     split_to_env_and_params \
@@ -556,7 +576,10 @@ function print_image_reports() {
         show_changes_env show_changes_params params_shift \
         "${@}"
     shift "${params_shift}"
-    oemids=( "${@}" )
+    get_batch_of_args oemids params_shift "${@}"
+    shift "${params_shift}"
+    get_batch_of_args base_sysexts params_shift "${@}"
+    shift "${params_shift}"
 
     flatcar_build_scripts_repo=$(realpath "${flatcar_build_scripts_repo}")
 
@@ -608,6 +631,25 @@ function print_image_reports() {
     echo "Note that vmlinuz-a also contains the kernel code, which might have changed too, so the reported difference does not accurately describe the change in initrd."
     echo
 
+    local base_sysext
+    for base_sysext in "${base_sysexts[@]}"; do
+        yell "Base sysext ${base_sysext} changes compared to ${previous_version_description}"
+        underline "Package updates, compared to ${previous_version_description}:"
+        env \
+            "${package_diff_env[@]}" FILE="rootfs-included-sysexts/${base_sysext}_packages.txt" \
+            "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
+
+        underline "Image file changes, compared to ${previous_version_description}:"
+        env \
+            "${package_diff_env[@]}" FILE="rootfs-included-sysexts/${base_sysext}_contents.txt" FILESONLY=1 CUTKERNEL=1 \
+            "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1
+
+        underline "Image file size changes, compared to ${previous_version_description}:"
+        if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:base-sysext-${base_sysext}-wtd}"; then
+            "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:base-sysext-${base_sysext}-old}" 2>&1
+        fi
+    done
+
     local oemid
     for oemid in "${oemids[@]}"; do
         yell "Sysext changes for OEM ${oemid} compared to ${previous_version_description}"
@@ -742,3 +784,23 @@ function split_to_env_and_params() {
     steap_to_shift_var_ref=${to_shift}
 }
 # --
+
+# 1 - name of an array variable that will contain the args
+# 2 - name of a scalar variable for shift number
+# @ - args with -- as batch separator
+function get_batch_of_args() {
+    local -n batch_ref=${1}; shift
+    local -n shift_ref=${1}; shift
+
+    batch_ref=()
+    shift_ref=0
+    local arg
+    for arg; do
+        shift_ref=$((shift_ref + 1))
+        if [[ ${arg} = '--' ]]; then
+            break
+        fi
+        batch_ref+=( "${arg}" )
+    done
+}
+# --

From c5b8a80d36f7eeebcba3e71818688a024f45bc54 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 26 Oct 2023 16:25:47 +0200
Subject: [PATCH 20/32] ci-automation: List built base sysext explicitly

Instead of depending on default value of build_image's base_sysext
parameter, create a file that explicitly lists which base sysexts will
be built for each architecture. The file can be sourced by other
scripts that need this kind of information. Currently, image.sh and
image_changes.sh use this file.
---
 .github/workflows/ci.yaml      |  2 +-
 ci-automation/base_sysexts.sh  | 16 ++++++++++++++++
 ci-automation/image.sh         |  5 +++++
 ci-automation/image_changes.sh | 13 +++++--------
 4 files changed, 27 insertions(+), 9 deletions(-)
 create mode 100644 ci-automation/base_sysexts.sh

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 01921d508a..d8b8cea8ad 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -283,7 +283,7 @@ jobs:
 
           declare -a oemids base_sysexts
           get_oem_id_list . "${arch}" oemids
-          get_base_sysext_list . base_sysexts
+          get_base_sysext_list . "${arch}" base_sysexts
           generate_image_changes_report \
               "${version_description}" 'image-changes-reports-release.txt' "../flatcar-build-scripts" \
               "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
diff --git a/ci-automation/base_sysexts.sh b/ci-automation/base_sysexts.sh
new file mode 100644
index 0000000000..7c58d574b1
--- /dev/null
+++ b/ci-automation/base_sysexts.sh
@@ -0,0 +1,16 @@
+# Definitions of base sysexts to be built, for each arch. Used by
+# image.sh and image_changes.sh.
+
+if [[ ${1:-} = 'local' ]]; then
+    local amd64_base_sysexts arm64_base_sysexts
+fi
+
+amd64_base_sysexts=(
+    'containerd-flatcar:app-containers/containerd'
+    'docker-flatcar:app-containers/docker'
+)
+
+arm64_base_sysexts=(
+    'containerd-flatcar:app-containers/containerd'
+    'docker-flatcar:app-containers/docker'
+)
diff --git a/ci-automation/image.sh b/ci-automation/image.sh
index 0cb5b7e118..5fc089354c 100644
--- a/ci-automation/image.sh
+++ b/ci-automation/image.sh
@@ -86,6 +86,10 @@ function _image_build_impl() {
 
     apply_local_patches
 
+    source ci-automation/base_sysexts.sh 'local'
+    local -n base_sysexts_ref="${arch}_base_sysexts"
+    local base_sysexts_param=$(export IFS=,; echo "${base_sysexts_ref[*]}")
+
     # build image and related artifacts
     ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
             -v "${vernum}" \
@@ -96,6 +100,7 @@ function _image_build_impl() {
     ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
             -v "${vernum}" \
             ./build_image --board="${arch}-usr" --group="${channel}" \
+                          --base_sysexts="${base_sysexts_param}" \
                           --output_root="${CONTAINER_IMAGE_ROOT}" \
                           --only_store_compressed \
                           prodtar container
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 5848d03e9a..cd229c724e 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -109,7 +109,7 @@ function image_changes() (
     echo
     local -a oemids base_sysexts
     get_oem_id_list . "${arch}" oemids
-    get_base_sysext_list . base_sysexts
+    get_base_sysext_list . "${arch}" base_sysexts
     generate_image_changes_report \
         "${version_description}" '-' "${fbs_repo}" \
         "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
@@ -262,18 +262,15 @@ function get_oem_id_list() {
 
 function get_base_sysext_list() {
     local scripts_repo=${1}; shift
+    local arch=${1}; shift
     local -n list_var_ref=${1}; shift
 
-    local line
-    line=$({ git -C "${scripts_repo}" grep -F 'DEFINE_string base_sysexts ' | head -n1; } || :)
-    line=${line#*'"'}
-    line=${line%'"'*}
+    source "${scripts_repo}/ci-automation/base_sysexts.sh" 'local'
+    local -n base_sysexts_ref="${arch}_base_sysexts"
 
     list_var_ref=()
-    local -a entries
-    mapfile -t entries <<<"${line//,/$'\n'}"
     local entry
-    for entry in "${entries[@]}"; do
+    for entry in "${base_sysexts_ref[@]}"; do
         list_var_ref+=( "${entry%%:*}" )
     done
 }

From f542807a9e9ca0287de96f4a5a229b40dc8f8461 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 26 Oct 2023 17:04:37 +0200
Subject: [PATCH 21/32] overlay, ci-automation: Factor out OEMID info to a
 separate file

Image changes job needs a list of OEMIDs that are built for a specific
architecture. Similar information already existed in the
coreos-base/common-oem-files ebuild, so factor it out to a separate
file, so the image changes job does not need to source the entire
ebuild (or process it in other way), but rather source the smaller
file.
---
 ci-automation/image_changes.sh                | 61 ++++---------------
 .../common-oem-files-0-r4.ebuild              | 17 +-----
 .../common-oem-files/files/oemids.sh          | 30 +++++++++
 3 files changed, 44 insertions(+), 64 deletions(-)
 create mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index cd229c724e..69c8c597b9 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -205,59 +205,22 @@ function git_tag_for_nightly() {
 # 2 - arch
 # 3 - name of an array variable to store the result in
 function get_oem_id_list() {
-    local scripts_repo arch
+    local scripts_repo arch list_var_name
     scripts_repo=${1}; shift
     arch=${1}; shift
-    local -n list_var_ref=${1}; shift
+    list_var_name=${1}; shift
 
-    local -a ebuilds
-    ebuilds=( "${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-"*'.ebuild' )
+    # This defines COMMON_OEMIDS, AMD64_ONLY_OEMIDS, ARM64_ONLY_OEMIDS
+    # and OEMIDS variable. We don't use the last one.
+    source "${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh" local
 
-    list_var_ref=()
-    if [[ ${#ebuilds[@]} -eq 0 ]]; then
-        return 0
-    fi
-    local mode
-    # 0 = no OEMIDS line found yet
-    # 1 = OEMIDS line found
-    mode=0
-    local -a fields
-    local first arch_field arch_found
-    while read -r -a fields; do
-        if [[ ${#fields[@]} -eq 0 ]]; then
-            continue
-        fi
-        first=${fields[0]}
-        case ${mode} in
-            0)
-                if [[ ${first} = 'OEMIDS=(' ]]; then
-                    mode=1
-                fi
-                ;;
-            1)
-                if [[ ${first} = ')' ]]; then
-                    break
-                fi
-                if [[ ${#fields[@]} -gt 1 ]]; then
-                    if [[ ${fields[1]} != '#' ]]; then
-                        echo "expect a line inside OEMIDS to be like '<OEMID> # <ARCH1> <ARCH2>…' or just '<OEMID>', got '${fields[*]}'" >&2
-                        exit 1
-                    fi
-                    arch_found=
-                    for arch_field in "${fields[@]:2}"; do
-                        if [[ ${arch} = "${arch_field}" ]]; then
-                            arch_found=x
-                            break
-                        fi
-                    done
-                    if [[ -z ${arch_found} ]]; then
-                        continue
-                    fi
-                fi
-                list_var_ref+=( "${first}" )
-                ;;
-        esac
-    done <"${ebuilds[0]}"
+    local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS"
+    local all_oemids=(
+        "${COMMON_OEMIDS[@]}"
+        "${arch_oemids_ref[@]}"
+    )
+
+    mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort)
 }
 
 function get_base_sysext_list() {
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
index 16b385bac9..33c4b20a73 100644
--- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
+++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
@@ -3,21 +3,8 @@
 
 EAPI=8
 
-# One OEM ID per line, a comment at the end of the line to denote
-# which arch this OEM is for (not necessary if OEM is built for all of
-# them). The arches should be space separated.
-#
-# This is used by the ci-automation/image_changes.sh script to figure
-# out the per-arch OEM IDs.
-OEMIDS=(
-    ami
-    azure
-    digitalocean # amd64
-    openstack
-    packet
-    qemu
-    vmware # amd64
-)
+# This defines the OEMIDS variable.
+source "${FILESDIR}/oemids.sh" only-oemids
 
 DESCRIPTION='Common OEM files'
 HOMEPAGE='https://www.flatcar.org/'
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh
new file mode 100644
index 0000000000..ca6b294aa9
--- /dev/null
+++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh
@@ -0,0 +1,30 @@
+if [[ ${1:-} = 'local' ]]; then
+    local -a COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS OEMIDS
+    shift
+fi
+
+COMMON_OEMIDS=(
+    ami
+    azure
+    openstack
+    packet
+    qemu
+)
+
+ARM64_ONLY_OEMIDS=(
+)
+
+AMD64_ONLY_OEMIDS=(
+    digitalocean
+    vmware
+)
+
+OEMIDS=(
+    "${COMMON_OEMIDS[@]}"
+    "${ARM64_ONLY_OEMIDS[@]}"
+    "${AMD64_ONLY_OEMIDS[@]}"
+)
+
+if [[ ${1:-} = 'only-oemids' ]]; then
+    unset COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS
+fi

From 8a75eba90628b3b53ca96872d50179b5fd92c367 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Wed, 1 Nov 2023 18:28:06 +0100
Subject: [PATCH 22/32] overlay, ci-automation: Another attempt at providing
 OEMID info

Please read the comment at the top of the coreos-base/common-oem-files
ebuild for details.
---
 ci-automation/image_changes.sh                |  8 ++-
 .../common-oem-files-0-r4.ebuild              | 60 ++++++++++++++++++-
 .../common-oem-files/files/oemids.sh          | 30 ----------
 3 files changed, 64 insertions(+), 34 deletions(-)
 delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 69c8c597b9..bc93b0d27a 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -210,9 +210,15 @@ function get_oem_id_list() {
     arch=${1}; shift
     list_var_name=${1}; shift
 
+    local -a ebuilds=("${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-"*'.ebuild')
+    if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then
+        echo "No coreos-base/common-oem-files ebuilds?!" >&2
+        exit 1
+    fi
+
     # This defines COMMON_OEMIDS, AMD64_ONLY_OEMIDS, ARM64_ONLY_OEMIDS
     # and OEMIDS variable. We don't use the last one.
-    source "${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh" local
+    source "${ebuilds[0]}" flatcar-local-variables
 
     local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS"
     local all_oemids=(
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
index 33c4b20a73..26c6c6ecb5 100644
--- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
+++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
@@ -1,10 +1,64 @@
 # Copyright (c) 2023 The Flatcar Maintainers.
 # Distributed under the terms of the GNU General Public License v2
 
-EAPI=8
+# This is a terrible hack done in order to avoid excessive duplication
+# of OEM IDs around the place. This ebuild basically serves as a
+# Gentoo ebuild (duh…) and as a bash file to be sourced in order to
+# get arch-specific information about possible OEM IDs. The latter
+# role is assumed when the ebuild is sourced with first argument being
+# 'flatcar-local-variables'. This role is used by our image-changes
+# job. All this fluff needs to happen before we define or invoke any
+# Gentoo-specific variables or functions like "EAPI" or "inherit" that
+# may mess up sourcing.
+#
+# This can't be done with a separate shell file in FILESDIR (I tried),
+# because portage moves the ebuild into some temporary directory where
+# FILESDIR, although defined, does not even exist. Probably a security
+# measure or something. So this needs to be done as
+# all-in-terrible-one (as opposed to all-in-wonder-one).
 
-# This defines the OEMIDS variable.
-source "${FILESDIR}/oemids.sh" only-oemids
+##
+## BEGIN HACK
+##
+
+if [[ ${1:-} = 'flatcar-local-variables' ]]; then
+    local -a COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS OEMIDS
+fi
+
+COMMON_OEMIDS=(
+    ami
+    azure
+    openstack
+    packet
+    qemu
+)
+
+ARM64_ONLY_OEMIDS=(
+)
+
+AMD64_ONLY_OEMIDS=(
+    digitalocean
+    vmware
+)
+
+OEMIDS=(
+    "${COMMON_OEMIDS[@]}"
+    "${ARM64_ONLY_OEMIDS[@]}"
+    "${AMD64_ONLY_OEMIDS[@]}"
+)
+
+if [[ ${1:-} = 'flatcar-local-variables' ]]; then
+    # Leave the sourced script here.
+    return 0
+else
+    unset COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS
+fi
+
+##
+## END HACK
+##
+
+EAPI=8
 
 DESCRIPTION='Common OEM files'
 HOMEPAGE='https://www.flatcar.org/'
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh
deleted file mode 100644
index ca6b294aa9..0000000000
--- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/oemids.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-if [[ ${1:-} = 'local' ]]; then
-    local -a COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS OEMIDS
-    shift
-fi
-
-COMMON_OEMIDS=(
-    ami
-    azure
-    openstack
-    packet
-    qemu
-)
-
-ARM64_ONLY_OEMIDS=(
-)
-
-AMD64_ONLY_OEMIDS=(
-    digitalocean
-    vmware
-)
-
-OEMIDS=(
-    "${COMMON_OEMIDS[@]}"
-    "${ARM64_ONLY_OEMIDS[@]}"
-    "${AMD64_ONLY_OEMIDS[@]}"
-)
-
-if [[ ${1:-} = 'only-oemids' ]]; then
-    unset COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS
-fi

From 9a780a3e9050c401820fdcf598ef1671a37939b3 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 2 Nov 2023 08:28:48 +0100
Subject: [PATCH 23/32] overlay coreos-base/common-oem-files: Move hack
 description down

EAPI needs to be defined within first 24 lines, so the description
would push the definition too much down as it needs to happen after
possible declaration of EAPI as local for the source case.
---
 .../common-oem-files-0-r4.ebuild              | 48 +++++++++----------
 1 file changed, 22 insertions(+), 26 deletions(-)

diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
index 26c6c6ecb5..2ec181eefd 100644
--- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
+++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
@@ -1,30 +1,14 @@
 # Copyright (c) 2023 The Flatcar Maintainers.
 # Distributed under the terms of the GNU General Public License v2
 
-# This is a terrible hack done in order to avoid excessive duplication
-# of OEM IDs around the place. This ebuild basically serves as a
-# Gentoo ebuild (duh…) and as a bash file to be sourced in order to
-# get arch-specific information about possible OEM IDs. The latter
-# role is assumed when the ebuild is sourced with first argument being
-# 'flatcar-local-variables'. This role is used by our image-changes
-# job. All this fluff needs to happen before we define or invoke any
-# Gentoo-specific variables or functions like "EAPI" or "inherit" that
-# may mess up sourcing.
-#
-# This can't be done with a separate shell file in FILESDIR (I tried),
-# because portage moves the ebuild into some temporary directory where
-# FILESDIR, although defined, does not even exist. Probably a security
-# measure or something. So this needs to be done as
-# all-in-terrible-one (as opposed to all-in-wonder-one).
-
-##
-## BEGIN HACK
-##
+# Hack description below.
 
 if [[ ${1:-} = 'flatcar-local-variables' ]]; then
-    local -a COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS OEMIDS
+    local -a EAPI COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS OEMIDS
 fi
 
+EAPI=8
+
 COMMON_OEMIDS=(
     ami
     azure
@@ -48,17 +32,29 @@ OEMIDS=(
 )
 
 if [[ ${1:-} = 'flatcar-local-variables' ]]; then
-    # Leave the sourced script here.
+    unset EAPI
     return 0
 else
     unset COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS
 fi
 
-##
-## END HACK
-##
-
-EAPI=8
+# The hack above was done in order to avoid excessive duplication of
+# OEM IDs around the place. This ebuild serves as a Gentoo ebuild and
+# as a bash file to be sourced in order to get arch-specific
+# information about possible OEM IDs. The latter role is assumed when
+# the ebuild is sourced with first argument being
+# 'flatcar-local-variables'. This role is used by our image-changes
+# job. All this fluff needs to happen before we define or invoke any
+# Gentoo-specific variables or functions like "DEPEND" or "inherit"
+# that may mess up sourcing. The only exception is EAPI, which must
+# happen in first 24 lines of the ebuild - this is defined in Package
+# Manager Specification and is enforced by portage.
+#
+# This can't be done with a separate shell file in FILESDIR, because
+# portage moves the ebuild into some temporary directory where
+# FILESDIR, although defined, does not even exist at first - it shows
+# up during the invocation of any src_ functions. Probably a security
+# measure or something.
 
 DESCRIPTION='Common OEM files'
 HOMEPAGE='https://www.flatcar.org/'

From 71eed26ee0633bbec766f8c10fe327ee166a2592 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 2 Nov 2023 09:50:35 +0100
Subject: [PATCH 24/32] overlay, ci-automation: Try making common-oem-files a
 valid ebuild

Assignment of EAPI must be the first non-comment, non-blank line in
the ebuild, otherwise portage masks it as corrupted.
---
 ci-automation/image_changes.sh                |  7 ++-
 .../common-oem-files-0-r4.ebuild              | 46 +++++++++----------
 2 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index bc93b0d27a..8fdba9d18f 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -216,8 +216,11 @@ function get_oem_id_list() {
         exit 1
     fi
 
-    # This defines COMMON_OEMIDS, AMD64_ONLY_OEMIDS, ARM64_ONLY_OEMIDS
-    # and OEMIDS variable. We don't use the last one.
+    # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS,
+    # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last
+    # one. Also defines global-by-default EAPI, which we make local
+    # here to avoid making it global.
+    local EAPI
     source "${ebuilds[0]}" flatcar-local-variables
 
     local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS"
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
index 2ec181eefd..67336f3219 100644
--- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
+++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0-r4.ebuild
@@ -1,14 +1,33 @@
 # Copyright (c) 2023 The Flatcar Maintainers.
 # Distributed under the terms of the GNU General Public License v2
 
-# Hack description below.
+EAPI=8
+
+# The hack below is there in order to avoid excessive duplication of
+# OEM IDs around the place. This ebuild serves as a Gentoo ebuild and
+# as a bash file to be sourced in order to get arch-specific
+# information about possible OEM IDs. The latter role is assumed when
+# the ebuild is sourced with first argument being
+# 'flatcar-local-variables'. Due to the requirements imposed by the
+# section 7.3.1 in Package Manager Specification (that says that EAPI
+# assignment must be the first non-comment, non-blank line in the
+# file), shell scripts wanting to source this ebuild for geting OEM
+# IDs, may need to declare EAPI as local, if it finds it suitable. The
+# role of sourced script is used by our image-changes job. All this
+# fluff needs to happen before we define or invoke any other
+# Gentoo-specific variables or functions like "DEPEND" or "inherit"
+# that may mess up sourcing.
+#
+# This can't be done with a separate shell file in FILESDIR, because
+# portage moves the ebuild into some temporary directory where
+# FILESDIR, although defined, does not even exist at first - it shows
+# up during the invocation of any src_ functions. Probably a security
+# measure or something.
 
 if [[ ${1:-} = 'flatcar-local-variables' ]]; then
-    local -a EAPI COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS OEMIDS
+    local -a COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS OEMIDS
 fi
 
-EAPI=8
-
 COMMON_OEMIDS=(
     ami
     azure
@@ -32,30 +51,11 @@ OEMIDS=(
 )
 
 if [[ ${1:-} = 'flatcar-local-variables' ]]; then
-    unset EAPI
     return 0
 else
     unset COMMON_OEMIDS ARM64_ONLY_OEMIDS AMD64_ONLY_OEMIDS
 fi
 
-# The hack above was done in order to avoid excessive duplication of
-# OEM IDs around the place. This ebuild serves as a Gentoo ebuild and
-# as a bash file to be sourced in order to get arch-specific
-# information about possible OEM IDs. The latter role is assumed when
-# the ebuild is sourced with first argument being
-# 'flatcar-local-variables'. This role is used by our image-changes
-# job. All this fluff needs to happen before we define or invoke any
-# Gentoo-specific variables or functions like "DEPEND" or "inherit"
-# that may mess up sourcing. The only exception is EAPI, which must
-# happen in first 24 lines of the ebuild - this is defined in Package
-# Manager Specification and is enforced by portage.
-#
-# This can't be done with a separate shell file in FILESDIR, because
-# portage moves the ebuild into some temporary directory where
-# FILESDIR, although defined, does not even exist at first - it shows
-# up during the invocation of any src_ functions. Probably a security
-# measure or something.
-
 DESCRIPTION='Common OEM files'
 HOMEPAGE='https://www.flatcar.org/'
 

From 0490c9c672b27634e473a36268d62ae9f17f8b21 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Wed, 8 Nov 2023 15:50:58 +0100
Subject: [PATCH 25/32] .github/ci: Report about base sysexts for against last
 nightly too

---
 .github/workflows/ci.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index d8b8cea8ad..cf4e46a3d0 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -339,14 +339,15 @@ jobs:
               'NEW_VERSION=HEAD'
           )
 
-          declare -a oemids
+          declare -a oemids base_sysexts
           get_oem_id_list . "${arch}" oemids
+          get_base_sysext_list . "${arch}" base_sysexts
           generate_image_changes_report \
-              "${version_description}" 'image-changes-reports-nightly.txt' "../flatcar-build-scripts" \
+              "${version_description}" 'image-changes-reports-release.txt' "../flatcar-build-scripts" \
               "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
               "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
               "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
-              "${oemids[@]}"
+              "${oemids[@]}" -- "${base_sysexts[@]}"
 
       - name: Upload binpkgs
         uses: actions/upload-artifact@v3

From 89f0cba474f76b700c72e3b0f72d384a23c13f35 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Thu, 9 Nov 2023 16:53:58 +0100
Subject: [PATCH 26/32] .github/ci: Deduplicate the image changes job

---
 .github/workflows/ci.yaml          | 114 +----------------------------
 .github/workflows/image_changes.sh |  72 ++++++++++++++++++
 2 files changed, 74 insertions(+), 112 deletions(-)
 create mode 100755 .github/workflows/image_changes.sh

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index cf4e46a3d0..40478f7d3c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -234,120 +234,10 @@ jobs:
           )
 
       - name: Generate reports against last release
-        shell: bash
-        run: |
-          set -euo pipefail
-          set -x
-
-          source ci-automation/image_changes.sh
-
-          git_tag=''
-          git_tag_for_release . git_tag
-          declare -a var_names=(
-              package_diff_env package_diff_params
-              size_changes_env size_changes_params
-              show_changes_env show_changes_params
-          )
-          declare -a "${var_names[@]}"
-          version_description=''
-          var_names+=( version_description )
-
-          prepare_env_vars_and_params_for_release "${arch}" "${git_tag}" "${var_names[@]}"
-
-          package_diff_env+=(
-              "FROM_B=file://${PWD}/artifacts/images"
-              # BOARD_B and CHANNEL_B are unused.
-          )
-          package_diff_params+=(
-              # The package-diff script appends version to the file
-              # URL, but the directory with the image has no version
-              # component at its end, so we use . as a version.
-              '.'
-          )
-          # Nothing to add to size changes env.
-          size_changes_params+=(
-              "local:${PWD}/artifacts/images"
-          )
-          show_changes_env+=(
-              # Override the default locations of repositories.
-              "SCRIPTS_REPO=."
-              "COREOS_OVERLAY_REPO=../coreos-overlay"
-              "PORTAGE_STABLE_REPO=../portage-stable"
-          )
-          show_changes_params+=(
-              # We may not have a tag handy, so we tell show-changes
-              # to use git HEAD as a reference to new changelog
-              # entries.
-              'NEW_VERSION=HEAD'
-          )
-
-          declare -a oemids base_sysexts
-          get_oem_id_list . "${arch}" oemids
-          get_base_sysext_list . "${arch}" base_sysexts
-          generate_image_changes_report \
-              "${version_description}" 'image-changes-reports-release.txt' "../flatcar-build-scripts" \
-              "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
-              "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
-              "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
-              "${oemids[@]}" -- "${base_sysexts[@]}"
+        run: .github/workflows/image_changes.sh release
 
       - name: Generate reports against last nightly
-        shell: bash
-        run: |
-          set -euo pipefail
-          set -x
-
-          source ci-automation/image_changes.sh
-
-          git_tag=''
-          git_tag_for_nightly . git_tag
-          declare -a var_names=(
-              package_diff_env package_diff_params
-              size_changes_env size_changes_params
-              show_changes_env show_changes_params
-          )
-          declare -a "${var_names[@]}"
-          version_description=''
-          var_names+=( version_description )
-
-          prepare_env_vars_and_params_for_bincache "${arch}" "${git_tag}" "${var_names[@]}"
-
-          package_diff_env+=(
-              "FROM_B=file://${PWD}/artifacts/images"
-              # BOARD_B and CHANNEL_B are unused.
-          )
-          package_diff_params+=(
-              # The package-diff script appends version to the file
-              # URL, but the directory with the image has no version
-              # component at its end, so we use . as a version.
-              '.'
-          )
-          # Nothing to add to size changes env.
-          size_changes_params+=(
-              "local:${PWD}/artifacts/images"
-          )
-          show_changes_env+=(
-              # Override the default locations of repositories.
-              "SCRIPTS_REPO=."
-              "COREOS_OVERLAY_REPO=../coreos-overlay"
-              "PORTAGE_STABLE_REPO=../portage-stable"
-          )
-          show_changes_params+=(
-              # We may not have a tag handy, so we tell show-changes
-              # to use git HEAD as a reference to new changelog
-              # entries.
-              'NEW_VERSION=HEAD'
-          )
-
-          declare -a oemids base_sysexts
-          get_oem_id_list . "${arch}" oemids
-          get_base_sysext_list . "${arch}" base_sysexts
-          generate_image_changes_report \
-              "${version_description}" 'image-changes-reports-release.txt' "../flatcar-build-scripts" \
-              "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
-              "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
-              "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
-              "${oemids[@]}" -- "${base_sysexts[@]}"
+        run: .github/workflows/image_changes.sh nightly
 
       - name: Upload binpkgs
         uses: actions/upload-artifact@v3
diff --git a/.github/workflows/image_changes.sh b/.github/workflows/image_changes.sh
new file mode 100755
index 0000000000..990a94eae5
--- /dev/null
+++ b/.github/workflows/image_changes.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+
+set -x
+set -euo pipefail
+
+mode=${1}; shift
+case ${mode} in
+    release|nightly)
+        :
+        ;;
+    *)
+        echo "invalid mode ${mode@Q}" >&2
+        exit 1
+        ;;
+esac
+
+git_tag_for_mode="git_tag_for_${mode}"
+prepare_env_vars_and_params_for_mode="prepare_env_vars_and_params_for_${mode}"
+report_file_name="image-changes-reports-${mode}.txt"
+
+source ci-automation/image_changes.sh
+
+git_tag=''
+"${git_tag_for_mode}" . git_tag
+
+declare -a var_names=(
+    package_diff_env package_diff_params
+    size_changes_env size_changes_params
+    show_changes_env show_changes_params
+)
+declare -a "${var_names[@]}"
+version_description=''
+var_names+=( version_description )
+
+"${prepare_env_vars_and_params_for_mode}" "${arch}" "${git_tag}" "${var_names[@]}"
+
+package_diff_env+=(
+    "FROM_B=file://${PWD}/artifacts/images"
+    # BOARD_B and CHANNEL_B are unused.
+)
+package_diff_params+=(
+    # The package-diff script appends version to the file
+    # URL, but the directory with the image has no version
+    # component at its end, so we use . as a version.
+    '.'
+)
+# Nothing to add to size changes env.
+size_changes_params+=(
+    "local:${PWD}/artifacts/images"
+)
+show_changes_env+=(
+    # Override the default locations of repositories.
+    "SCRIPTS_REPO=."
+    "COREOS_OVERLAY_REPO=../coreos-overlay"
+    "PORTAGE_STABLE_REPO=../portage-stable"
+)
+show_changes_params+=(
+    # We may not have a tag handy, so we tell show-changes
+    # to use git HEAD as a reference to new changelog
+    # entries.
+    'NEW_VERSION=HEAD'
+)
+
+declare -a oemids base_sysexts
+get_oem_id_list . "${arch}" oemids
+get_base_sysext_list . "${arch}" base_sysexts
+generate_image_changes_report \
+    "${version_description}" "${report_file_name}" "../flatcar-build-scripts" \
+    "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
+    "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
+    "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
+    "${oemids[@]}" -- "${base_sysexts[@]}"

From 810306c884cfc1a1561d5b3a5ba6c0ba05994101 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Fri, 10 Nov 2023 11:36:10 +0100
Subject: [PATCH 27/32] ci-automation/image-changes, .github/ci: Further
 deduplication

---
 .github/workflows/ci.yaml          |   4 +-
 .github/workflows/image_changes.sh |  97 ++++++++--------------
 ci-automation/image_changes.sh     | 126 +++++++++++++++++++----------
 3 files changed, 118 insertions(+), 109 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 40478f7d3c..41b5465e21 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -234,10 +234,10 @@ jobs:
           )
 
       - name: Generate reports against last release
-        run: .github/workflows/image_changes.sh release
+        run: .github/workflows/image_changes.sh ${{ matrix.arch }} release
 
       - name: Generate reports against last nightly
-        run: .github/workflows/image_changes.sh nightly
+        run: .github/workflows/image_changes.sh ${{ matrix.arch }} nightly
 
       - name: Upload binpkgs
         uses: actions/upload-artifact@v3
diff --git a/.github/workflows/image_changes.sh b/.github/workflows/image_changes.sh
index 990a94eae5..07e893db6f 100755
--- a/.github/workflows/image_changes.sh
+++ b/.github/workflows/image_changes.sh
@@ -3,70 +3,41 @@
 set -x
 set -euo pipefail
 
-mode=${1}; shift
-case ${mode} in
-    release|nightly)
-        :
-        ;;
-    *)
-        echo "invalid mode ${mode@Q}" >&2
-        exit 1
-        ;;
-esac
-
-git_tag_for_mode="git_tag_for_${mode}"
-prepare_env_vars_and_params_for_mode="prepare_env_vars_and_params_for_${mode}"
-report_file_name="image-changes-reports-${mode}.txt"
-
 source ci-automation/image_changes.sh
 
-git_tag=''
-"${git_tag_for_mode}" . git_tag
+# Callback invoked by run_image_changes_job, read its docs to learn
+# about the details about the callback.
+function github_ricj_callback() {
+    package_diff_env+=(
+        "FROM_B=file://${PWD}/artifacts/images"
+        # BOARD_B and CHANNEL_B are unused.
+    )
+    package_diff_params+=(
+        # The package-diff script appends version to the file
+        # URL, but the directory with the image has no version
+        # component at its end, so we use . as a version.
+        '.'
+    )
+    # Nothing to add to size changes env.
+    size_changes_params+=(
+        "local:${PWD}/artifacts/images"
+    )
+    show_changes_env+=(
+        # Override the default locations of repositories.
+        "SCRIPTS_REPO=."
+        "COREOS_OVERLAY_REPO=../coreos-overlay"
+        "PORTAGE_STABLE_REPO=../portage-stable"
+    )
+    show_changes_params+=(
+        # We may not have a tag handy, so we tell show-changes
+        # to use git HEAD as a reference to new changelog
+        # entries.
+        'NEW_VERSION=HEAD'
+    )
+}
 
-declare -a var_names=(
-    package_diff_env package_diff_params
-    size_changes_env size_changes_params
-    show_changes_env show_changes_params
-)
-declare -a "${var_names[@]}"
-version_description=''
-var_names+=( version_description )
+arch=${1}; shift
+mode=${1; shift
+report_file_name="image-changes-reports-${mode}.txt"
 
-"${prepare_env_vars_and_params_for_mode}" "${arch}" "${git_tag}" "${var_names[@]}"
-
-package_diff_env+=(
-    "FROM_B=file://${PWD}/artifacts/images"
-    # BOARD_B and CHANNEL_B are unused.
-)
-package_diff_params+=(
-    # The package-diff script appends version to the file
-    # URL, but the directory with the image has no version
-    # component at its end, so we use . as a version.
-    '.'
-)
-# Nothing to add to size changes env.
-size_changes_params+=(
-    "local:${PWD}/artifacts/images"
-)
-show_changes_env+=(
-    # Override the default locations of repositories.
-    "SCRIPTS_REPO=."
-    "COREOS_OVERLAY_REPO=../coreos-overlay"
-    "PORTAGE_STABLE_REPO=../portage-stable"
-)
-show_changes_params+=(
-    # We may not have a tag handy, so we tell show-changes
-    # to use git HEAD as a reference to new changelog
-    # entries.
-    'NEW_VERSION=HEAD'
-)
-
-declare -a oemids base_sysexts
-get_oem_id_list . "${arch}" oemids
-get_base_sysext_list . "${arch}" base_sysexts
-generate_image_changes_report \
-    "${version_description}" "${report_file_name}" "../flatcar-build-scripts" \
-    "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
-    "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
-    "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
-    "${oemids[@]}" -- "${base_sysexts[@]}"
+run_image_changes_job "${arch}" "${mode}" "${report_file_name}" '../flatcar-build-scripts' github_ricj_callback
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 8fdba9d18f..498f2efabd 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -36,36 +36,28 @@ function image_changes() (
     local arch what
 
     arch=${1}; shift
-    what=${1}; shift
+    # make nightly and release from last-nightly and last-release, respectively
+    mode=${1#last-}; shift
 
-    local -a package_diff_env package_diff_params
-    local -a size_changes_env size_changes_params
-    local -a show_changes_env show_changes_params
-    local version_description
-    local -a var_names=(
-        package_diff_env package_diff_params
-        size_changes_env size_changes_params
-        show_changes_env show_changes_params
-        version_description
-    )
-
-    case ${what} in
-        last-release)
-            local git_tag
-            git_tag_for_release . git_tag
-            prepare_env_vars_and_params_for_release "${arch}" "${git_tag}" "${var_names[@]}"
-            ;;
-        last-nightly)
-            local git_tag
-            git_tag_for_nightly . git_tag
-            prepare_env_vars_and_params_for_bincache "${arch}" "${git_tag}" "${var_names[@]}"
-            ;;
-        *)
-            echo "invalid argument '${what}', expected 'last-nightly' or 'last-release'" >&2
-            exit 1
-            ;;
-    esac
+    local fbs_repo='../flatcar-build-scripts'
+    rm -rf "${fbs_repo}"
+    git clone \
+        --depth 1 \
+        --single-branch \
+        "https://github.com/flatcar/flatcar-build-scripts" \
+        "${fbs_repo}"
+    if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then
+        local BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}")
+    fi
+    echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${version}/flatcar_production_image.bin.bz2"
+    echo
+    run_image_changes_job "${arch}" "${mode}" '-' "${fbs_repo}" ricj_callback
+)
+# --
 
+# Callback invoked by run_image_changes_job, read its docs to learn
+# about the details about the callback.
+function ricj_callback() {
     local ic_head_tag version
     head_git_tag . ic_head_tag
     version=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
@@ -94,29 +86,75 @@ function image_changes() (
         # here instead of the vernum variable.
         "NEW_VERSION=${ic_head_tag}"
     )
+}
+# --
+
+# Runs the whole image changes job for given arch and mode. The report
+# is written to the given file. The reports will be done using tools
+# from the passed path to the flatcar build scripts repository. The
+# parameters and environment of the tools should will be partially set
+# up depending on mode, but the further setup should be done by the
+# passed callback.
+#
+# The callback takes no parameters. It should assume that array
+# variables 'package_diff_env', 'package_diff_params',
+# 'size_changes_env', 'size_changes_params', 'show_changes_env' and
+# 'show_changes_params' are already defined, so it can append
+# necessary data into them.
+#
+# 1 - arch
+# 2 - mode
+# 3 - report file name ('-' for standard output)
+# 4 - path to the flatcar-build-scripts repository
+# 5 - name of a callback function
+function run_image_changes_job() {
+    arch=${1}; shift
+    mode=${1}; shift
+    report_file_name=${1}; shift
+    fbs_repo=${1}; shift
+    cb=${1}; shift
+
+    case ${mode} in
+        release|nightly)
+            :
+            ;;
+        *)
+            echo "invalid mode ${mode@Q}, expected 'nightly' or 'release'" >&2
+            exit 1
+            ;;
+    esac
+
+    local -a package_diff_env package_diff_params
+    local -a size_changes_env size_changes_params
+    local -a show_changes_env show_changes_params
+    local version_description
+    local -a var_names=(
+        package_diff_env package_diff_params
+        size_changes_env size_changes_params
+        show_changes_env show_changes_params
+        version_description
+    )
+    local git_tag_for_mode prepare_env_vars_and_params_for_mode
+    git_tag_for_mode="git_tag_for_${mode}"
+    prepare_env_vars_and_params_for_mode="prepare_env_vars_and_params_for_${mode}"
+
+    local git_tag
+    "${git_tag_for_mode}" . git_tag
+    "${prepare_env_vars_and_params_for_mode}" "${arch}" "${git_tag}" "${var_names[@]}"
+
+    # invoke callback that should append necessary info to env and params variables
+    "${cb}"
 
-    local fbs_repo='../flatcar-build-scripts'
-    rm -rf "${fbs_repo}"
-    git clone \
-        --depth 1 \
-        --single-branch \
-        "https://github.com/flatcar/flatcar-build-scripts" \
-        "${fbs_repo}"
-    if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then
-        local BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}")
-    fi
-    echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${version}/flatcar_production_image.bin.bz2"
-    echo
     local -a oemids base_sysexts
     get_oem_id_list . "${arch}" oemids
     get_base_sysext_list . "${arch}" base_sysexts
     generate_image_changes_report \
-        "${version_description}" '-' "${fbs_repo}" \
+        "${version_description}" "${report_file_name}" "${fbs_repo}" \
         "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \
         "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \
         "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \
         "${oemids[@]}" -- "${base_sysexts[@]}"
-)
+}
 # --
 
 # Gets a git tag that can be passed to
@@ -180,7 +218,7 @@ function head_git_tag() {
 }
 
 # Gets a git tag of a previous nightly that can be passed to
-# prepare_env_vars_and_params_for_bincache.
+# prepare_env_vars_and_params_for_nightly.
 #
 # 1 - scripts repo
 # 2 - name of a variable to store the result in
@@ -357,7 +395,7 @@ function prepare_env_vars_and_params_for_release() {
 # nightly relative to the git tag. The git tag should be in form of
 # <channel>-<version id>-<build id>, which is the usual format used in
 # scripts repo.
-function prepare_env_vars_and_params_for_bincache() {
+function prepare_env_vars_and_params_for_nightly() {
     local arch git_tag
     arch=${1}; shift
     git_tag=${1}; shift

From c9b9864ae17e4c78ca38570c375465c6960213a6 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Wed, 15 Nov 2023 18:01:33 +0100
Subject: [PATCH 28/32] .github/ci: AAAARGH!

---
 .github/workflows/image_changes.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/image_changes.sh b/.github/workflows/image_changes.sh
index 07e893db6f..37a20318c1 100755
--- a/.github/workflows/image_changes.sh
+++ b/.github/workflows/image_changes.sh
@@ -37,7 +37,7 @@ function github_ricj_callback() {
 }
 
 arch=${1}; shift
-mode=${1; shift
+mode=${1}; shift
 report_file_name="image-changes-reports-${mode}.txt"
 
 run_image_changes_job "${arch}" "${mode}" "${report_file_name}" '../flatcar-build-scripts' github_ricj_callback

From 00dad31c11546038d32fd54136ebb07d8d0544f2 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Wed, 15 Nov 2023 18:06:16 +0100
Subject: [PATCH 29/32] ci-automation/image-changes: Meh

---
 ci-automation/image_changes.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index 498f2efabd..adbbf86274 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -49,6 +49,8 @@ function image_changes() (
     if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then
         local BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}")
     fi
+    local version
+    version=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
     echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${version}/flatcar_production_image.bin.bz2"
     echo
     run_image_changes_job "${arch}" "${mode}" '-' "${fbs_repo}" ricj_callback

From a39e53aed3498700752fa9867231b5859accacfb Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Mon, 20 Nov 2023 11:56:22 +0100
Subject: [PATCH 30/32] .github/ci: Install python3-packaging package

show-fixed-kernel-cves.py script from flatcar-build-scripts requires
this package:

Traceback (most recent call last):
  File "/home/runner/actions-runner/_work/scripts/scripts/flatcar-build-scripts/show-fixed-kernel-cves.py", line 29, in <module>
    from packaging import version
ModuleNotFoundError: No module named 'packaging'
---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 41b5465e21..57d0d63e25 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -55,7 +55,7 @@ jobs:
           sudo rm /bin/sh
           sudo ln -s /bin/bash /bin/sh
           sudo apt-get update
-          sudo apt-get install -y ca-certificates curl git gnupg lsb-release python3 qemu-user-static zstd
+          sudo apt-get install -y ca-certificates curl git gnupg lsb-release python3 python3-packaging qemu-user-static zstd
           sudo mkdir -p /etc/apt/keyrings
           curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
           echo \

From a87a8821fe63913b59f6f187393b5bacac9e456a Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Tue, 21 Nov 2023 15:45:36 +0100
Subject: [PATCH 31/32] ci-automation/image-changes: Avoid swallowing errors

---
 ci-automation/image.sh         |  3 ++-
 ci-automation/image_changes.sh | 26 +++++++++++++++++---------
 2 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/ci-automation/image.sh b/ci-automation/image.sh
index 5fc089354c..5f9c9c252f 100644
--- a/ci-automation/image.sh
+++ b/ci-automation/image.sh
@@ -88,7 +88,8 @@ function _image_build_impl() {
 
     source ci-automation/base_sysexts.sh 'local'
     local -n base_sysexts_ref="${arch}_base_sysexts"
-    local base_sysexts_param=$(export IFS=,; echo "${base_sysexts_ref[*]}")
+    local base_sysexts_param
+    base_sysexts_param=$(export IFS=,; echo "${base_sysexts_ref[*]}")
 
     # build image and related artifacts
     ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index adbbf86274..a036d52a89 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -47,7 +47,8 @@ function image_changes() (
         "https://github.com/flatcar/flatcar-build-scripts" \
         "${fbs_repo}"
     if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then
-        local BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}")
+        local BUILDCACHE_SERVER
+        BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}")
     fi
     local version
     version=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}")
@@ -522,14 +523,21 @@ function channel_version() (
     local channel=${1}; shift
     local board=${1}; shift
 
-    source <(curl \
-                 -fsSL \
-                 --retry-delay 1 \
-                 --retry 60 \
-                 --retry-connrefused \
-                 --retry-max-time 60 \
-                 --connect-timeout 20 \
-                 "https://${channel}.release.flatcar-linux.net/${board}/current/version.txt")
+    local tmp_version_txt
+    tmp_version_txt=$(mktemp)
+    # This function runs in a subshell, so we can have our own scoped
+    # traps.
+    trap 'rm "${tmp_version_txt}"' EXIT
+
+    curl \
+        -fsSL \
+        --retry-delay 1 \
+        --retry 60 \
+        --retry-connrefused \
+        --retry-max-time 60 \
+        --connect-timeout 20 \
+        "https://${channel}.release.flatcar-linux.net/${board}/current/version.txt" >"${tmp_version_txt}"
+    source "${tmp_version_txt}"
     echo "${FLATCAR_VERSION}"
 )
 # --

From 3d4ddf86456c92ec4482e3f03bdfd7e64ac47f10 Mon Sep 17 00:00:00 2001
From: Krzesimir Nowak <knowak@microsoft.com>
Date: Tue, 21 Nov 2023 15:59:20 +0100
Subject: [PATCH 32/32] ci-automation/base-sysexts: Simplify

No need in making it arch-specific.
---
 ci-automation/base_sysexts.sh  | 13 ++++---------
 ci-automation/image.sh         |  3 +--
 ci-automation/image_changes.sh |  3 +--
 3 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/ci-automation/base_sysexts.sh b/ci-automation/base_sysexts.sh
index 7c58d574b1..d5afeac5ea 100644
--- a/ci-automation/base_sysexts.sh
+++ b/ci-automation/base_sysexts.sh
@@ -1,16 +1,11 @@
-# Definitions of base sysexts to be built, for each arch. Used by
-# image.sh and image_changes.sh.
+# Definitions of base sysexts to be built. Used by image.sh and
+# image_changes.sh.
 
 if [[ ${1:-} = 'local' ]]; then
-    local amd64_base_sysexts arm64_base_sysexts
+    local ciabs_base_sysexts
 fi
 
-amd64_base_sysexts=(
-    'containerd-flatcar:app-containers/containerd'
-    'docker-flatcar:app-containers/docker'
-)
-
-arm64_base_sysexts=(
+ciabs_base_sysexts=(
     'containerd-flatcar:app-containers/containerd'
     'docker-flatcar:app-containers/docker'
 )
diff --git a/ci-automation/image.sh b/ci-automation/image.sh
index 5f9c9c252f..139956970c 100644
--- a/ci-automation/image.sh
+++ b/ci-automation/image.sh
@@ -87,9 +87,8 @@ function _image_build_impl() {
     apply_local_patches
 
     source ci-automation/base_sysexts.sh 'local'
-    local -n base_sysexts_ref="${arch}_base_sysexts"
     local base_sysexts_param
-    base_sysexts_param=$(export IFS=,; echo "${base_sysexts_ref[*]}")
+    base_sysexts_param=$(export IFS=,; echo "${ciabs_base_sysexts[*]}")
 
     # build image and related artifacts
     ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh
index a036d52a89..372c2d0819 100644
--- a/ci-automation/image_changes.sh
+++ b/ci-automation/image_changes.sh
@@ -279,11 +279,10 @@ function get_base_sysext_list() {
     local -n list_var_ref=${1}; shift
 
     source "${scripts_repo}/ci-automation/base_sysexts.sh" 'local'
-    local -n base_sysexts_ref="${arch}_base_sysexts"
 
     list_var_ref=()
     local entry
-    for entry in "${base_sysexts_ref[@]}"; do
+    for entry in "${ciabs_base_sysexts[@]}"; do
         list_var_ref+=( "${entry%%:*}" )
     done
 }