Support OEM systemd-sysext images and Flatcar extensions

The vendor tools on the OEM partition weren't updated. We now want to
ship them as systemd-sysext images which we can easily update. This
change extends the Flatcar A/B update mechanism to cover the OEM
systemd-sysext images. The same mechanism is also able to support
"official" Flatcar extensions, e.g., a ZFS extension.

build_library/vm_image_util.sh

@@ -531,12 +531,7 @@ install_oem_sysext() {
     local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext"
     local built_sysext_filename="${oem_sysext}.raw"
     local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}"
-    # TODO: Set 'version' to "${FLATCAR_VERSION}" and drop
-    # VERSION_FIELD_OVERRIDE when we implement updating OEM sysexts.
-    local version='initial'
-    local build_sysext_env=(
-        VERSION_FIELD_OVERRIDE='SYSEXT_LEVEL=1.0'
-    )
+    local version='${FLATCAR_VERSION}'
     local metapkg="coreos-base/${oem_sysext}"
     local build_sysext_flags=(
         --board="${BOARD}"

changelog/changes/2023-09-06-oem-vendor-tool-updates.md

@@ -0,0 +1 @@
+- OEM vendor tools are now A/B updated if they are shipped as systemd-sysext images, the migration happens when both partitions require a systemd-sysext OEM image - note that this will delete the `nvidia.service` from `/etc` on Azure because it's now part of `/usr` ([Flatcar#60](https://github.com/flatcar/Flatcar/issues/60))

ci-automation/vendor-testing/qemu_update.sh

@@ -13,11 +13,11 @@ set -euo pipefail
 source ci-automation/vendor_test.sh
 
 # The last check is not perfect (if both tests are rerun, it will only look at the name of the second test) but hopefully still good enough to prevent wrong usage
-if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update.payload" ]]; then
+if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update."* ]]; then
     echo "1..1" > "${CIA_TAPFILE}"
     echo "not ok - all qemu update tests" >> "${CIA_TAPFILE}"
     echo "  ---" >> "${CIA_TAPFILE}"
-    echo "  ERROR: Only cl.update.payload is supported, got '$*'." | tee -a "${CIA_TAPFILE}"
+    echo "  ERROR: Only cl.update.payload and cl.update.oem are supported, got '$*'." | tee -a "${CIA_TAPFILE}"
     echo "  ..." >> "${CIA_TAPFILE}"
     break_retest_cycle
     exit 1
@@ -87,11 +87,16 @@ query_kola_tests() {
 run_kola_tests() {
     local instance_type="${1}"; shift;
     local instance_tapfile="${1}"; shift
+    local tests=("cl.update.payload")
     local image
     if [ "${instance_type}" = "previous" ]; then
         image="tmp/flatcar_production_image_previous.bin"
     elif [ "${instance_type}" = "first_dual" ]; then
         image="tmp/flatcar_production_image_first_dual.bin"
+        # Only run this test if the Azure dev payload exists on bincache because the fallback download needs it
+        if curl --head -o /dev/null -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://bincache.flatcar-linux.net/images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update-oem-azure.gz"; then
+          tests+=("cl.update.oem")
+        fi
     else
         echo "Wrong instance type ${instance_type}" >&2
         exit 1
@@ -107,7 +112,7 @@ run_kola_tests() {
       --torcx-manifest="${CIA_TORCX_MANIFEST}" \
       --update-payload="${QEMU_UPDATE_PAYLOAD}" \
       ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
-      cl.update.payload
+      "${tests[@]}"
 }
 
 run_kola_tests_on_instances "previous" "${CIA_TAPFILE}" "${CIA_FIRST_RUN}" first_dual -- cl.update.payload -- "${@}"

sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild

@@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64 ~arm ~arm64 ~x86"
 else
-	CROS_WORKON_COMMIT="235b6141a0b6eb4c1e4dc7778a896f22756898a3" # flatcar-master
+	CROS_WORKON_COMMIT="a7c8b0c7ba4209e520532d1b92131e8ad23c40fb" # flatcar-master
 	KEYWORDS="amd64 arm arm64 x86"
 fi
 

sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/files/oems/azure

@@ -0,0 +1,13 @@
+/etc/systemd/system/nvidia.service
+/oem/bin/setup-nvidia
+/oem/bin/install-nvidia
+/oem/units/nvidia.service
+/etc/systemd/system/oem-cloudinit.service
+/etc/systemd/system/multi-user.target.wants/oem-cloudinit.service
+/etc/systemd/system/waagent.service
+/etc/systemd/system/multi-user.target.wants/waagent.service
+/oem/waagent.conf
+/oem/python/
+/oem/bin/
+/oem/units/
+/oem/base/

sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/misc-files-0.ebuild

@@ -79,6 +79,14 @@ src_install() {
     insinto '/etc/bash/bashrc.d'
     doins "${FILESDIR}/99-flatcar-bcc"
 
+    insinto '/usr/share/flatcar'
+    # The "oems" folder should contain a file "$OEMID" for each expected OEM sysext and
+    # either be empty or contain a newline-separated list of files to delete during the
+    # migration (done from the initrd). The existence of the file will help old clients
+    # to do the fallback download of the sysext payload in the postinstall hook.
+    # The paths should use /oem instead of /usr/share/oem/ to avoid symlink resolution.
+    doins -r "${FILESDIR}"/oems
+
     dotmpfiles "${T}/home-core-bash-symlinks.conf"
     # Ideally we would be calling systemd-tmpfiles to create the
     # symlinks, but at this point systemd may not have any info about

sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-9999.ebuild

@@ -8,7 +8,7 @@ CROS_WORKON_REPO="https://github.com"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64 ~arm ~arm64 ~x86"
 else
-	CROS_WORKON_COMMIT="c6f566d47d8949632f7f43871eb8d5c625af3209" # flatcar-master
+	CROS_WORKON_COMMIT="b3cdd63f96cfa35c73c56dc6ce4fbe6cbc5d6740" # flatcar-master
 	KEYWORDS="amd64 arm64"
 fi
 
@@ -102,6 +102,11 @@ src_install() {
 	insinto /usr/share/dbus-1/system.d
 	doins com.coreos.update1.conf
 
+	insinto /usr/share/update_engine
+	doins src/update_engine/update_metadata.proto
+	exeinto /usr/share/update_engine
+	doexe decode_payload
+
 	# Install rule to remove old UpdateEngine.conf from /etc
 	dotmpfiles "${FILESDIR}"/update-engine.conf
 }

sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild

@@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64 ~arm ~arm64 ~x86"
 else
-	CROS_WORKON_COMMIT="cc0fdec0cc6c5692acac95a928984ea8a5eb8f08" # flatcar-master
+	CROS_WORKON_COMMIT="1066c7f5155f9846da3f83473be9c8768524b52c" # flatcar-master
 	KEYWORDS="amd64 arm arm64 x86"
 fi