WIP: initial stab at a sysext dev container

WARNING. This is a WIP commit that will be amended and force-pushed.
Please use a separate branch if you want to work on this.

Build with (e.g.)

sudo ./build_sysext --mangle_with_base --manglefs_script build_library/sysext_mangle_devext --manglefs_script_args "https://bincache.flatcar-linux.net/boards/amd64-usr/4117.0.0+nightly-20241008-2100/pkgs/" devext coreos-base/coreos-dev

TODO:
- make profile work

Signed-off-by: Thilo Fromm <thilofromm@microsoft.com>

build_image

@@ -30,9 +30,9 @@ DEFINE_string getbinpkgver "" \
 DEFINE_boolean enable_rootfs_verification ${FLAGS_TRUE} \
   "Default all bootloaders to use kernel-based root fs integrity checking."
 DEFINE_string base_pkg "coreos-base/coreos" \
-  "The base portage package to base the build off of (only applies to prod images)"
+  "The base portage package to base the OS image build off of"
 DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
-  "The base portage package to base the build off of (only applies to dev containers)"
+  "The base portage package to base the development container / sysext build off of"
 DEFINE_string base_sysexts "containerd-flatcar:app-containers/containerd,docker-flatcar:app-containers/docker&app-containers/docker-cli&app-containers/docker-buildx" \
   "Comma-separated list of name:package[&package[&package]] - build 'package' (a single package or a list of packages separated by '&') into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
@@ -60,10 +60,11 @@ different forms.  This scripts can be used to build the following:
 prod - Production image for CoreOS. This image is for booting (default if no argument is given).
 prodtar - Production container tar ball (implies prod). This can e.g. be used to run the Flatcar production image as a container (run machinectl import-tar or docker import).
 container - Developer image with single filesystem, bootable by nspawn.
+devext - Developer system extension including compiler and packaging tools.
 
 Examples:
 
-build_image --board=<board> [prod] [prodtar] [container] - builds developer and production images/tars.
+build_image --board=<board> [prod] [prodtar] [container] [devext] - builds developer and production images/tars.
 ...
 "
 show_help_if_requested "$@"
@@ -109,12 +110,14 @@ fi
 PROD_IMAGE=0
 PROD_TAR=0
 CONTAINER=0
+DEVEXT=0
 SYSEXT=0
 for arg in "$@"; do
   case "${arg}" in
     prod) PROD_IMAGE=1 ;;
     prodtar) PROD_IMAGE=1 PROD_TAR=1 ;;
     container) CONTAINER=1 ;;
+    devext) DEVEXT=1 ;;
     sysext) SYSEXT=1 ;;
     *)    die_notrace "Unknown image type ${arg}" ;;
   esac
@@ -168,6 +171,21 @@ if [[ "${CONTAINER}" -eq 1 ]]; then
   create_dev_container "${FLATCAR_DEVELOPER_CONTAINER_NAME}" "${CONTAINER_LAYOUT}" "${FLAGS_devcontainer_binhost}" "${FLAGS_group}" ${FLAGS_base_dev_pkg}
 fi
 
+# FIXME: glibc, linux-headers, and readline aren't properly installed
+#        on the base image (lack development files) so we need to force a re-install.
+if [[ "${DEVEXT}" -eq 1 ]]; then
+  sudo ${SCRIPT_ROOT}/build_sysext --mangle_with_base \
+                                   --preparefs_script build_library/sysext_mangle_devext \
+                                   --preparefs_script_args prepare \
+                                   --manglefs_script build_library/sysext_mangle_devext \
+                                   --manglefs_script_args "${FLAGS_devcontainer_binhost}" \
+                                   --remove_duplicate_files \
+                                   devext \
+                                   @system "${FLAGS_base_dev_pkg}"
+fi
+
+# sys-devel/gcc glibc linux-headers readline bzip
+
 if [[ "${PROD_IMAGE}" -eq 1 ]]; then
   IMAGE_BUILD_TYPE="prod"
   create_prod_image ${FLATCAR_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} ${FLAGS_base_pkg} ${FLAGS_base_sysexts}

build_library/sysext_mangle_devext

@@ -0,0 +1,102 @@
+#!/bin/bash
+# Copyright (c) 2024 by the Flatcar Maintainers.
+# Use of this source code is governed by the Apache 2.0 license.
+
+# Mangler script for the development sysext.
+# Called by build_image -> build_sysext.
+#
+# Mandatory positional arguments
+# 1. image_root  - root directory of the image (sysext + rootfs).
+#                  Passed by build_sysext.
+# 2. binhost_url - complete URL for binpackages.
+#                  Passed via --manglefs_script_args. 
+
+image_root="${1}"
+binhost_url="${2}"
+
+devext_basedir="/usr/share/flatcar/devext"
+
+# We're in build_library/, script root is one up
+SCRIPT_ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../"; pwd)"
+. "${SCRIPT_ROOT}/common.sh" || exit 1
+
+# Script must run inside the chroot
+assert_inside_chroot
+switch_to_strict_mode
+
+# run_ldconfig, run_localedef
+source "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1
+# get_board_*
+source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
+
+info "running ldconfig, localedef."
+ARCH=$(get_board_arch $BOARD)
+CHOST=$(get_board_chost $BOARD)
+export ARCH CHOST
+
+
+pushd "${image_root}"
+
+# Prepare or mangle?
+if [[ "$2" = "prepare" ]] ; then
+    # We're running _before_ emerge is called
+
+    # Purge all package info to force a re-install.
+    # build_sysext will have been instructed to remove duplicates.
+    rm -rf ./var/db/pkg
+
+    info "prepare: Setting up dev profile"
+
+    # Remove base OS' package.provided, soname.provided
+    rm ./etc/portage/profile/*.provided
+
+    profile="./etc/portage/make.profile"
+    dev="$(readlink -f "${profile}")/dev"
+    rm -vf "${profile}"
+    ln -vs "${dev}" "${profile}"
+    exit 0
+fi
+
+# We're running _after_ emerge ran.
+
+run_ldconfig .
+run_localedef .
+
+files_dir="${SRC_ROOT}/third_party/coreos-overlay/coreos/sysext/devext"
+info "Installing extra files from '${files_dir}' to '${devext_basedir}'"
+# ATTENTION: don't preserve ownership as repo is owned by sdk user
+cp -vdR --preserve=mode,timestamps "${files_dir}/"* "."
+
+# OS image does not include 
+# Force an upcopy so it is preserved in the sysext.
+info "preserving /usr/include"
+find ./usr/include -exec sh -c 'touch {}' \;
+
+info "capturing portage state."
+mkdir -p .${devext_basedir}/var/db \
+         .${devext_basedir}/var/lib/portage \
+         .${devext_basedir}/etc/portage/repos.conf
+
+cp -R --dereference ./var/db/pkg .${devext_basedir}/var/db/
+cp -R --dereference ./etc/sandbox* .${devext_basedir}/etc
+
+info "copying ebuild repos."
+for src in portage-stable coreos-overlay; do
+	info " repo '${src}'"
+    mkdir -p ".${devext_basedir}/var/lib/portage/"
+    cp -R "${SRC_ROOT}/third_party/${src}" \
+          ".${devext_basedir}/var/lib/portage/"
+done
+
+export PORTAGE_BINHOST="${binhost_url}"
+envsubst >".${devext_basedir}/etc/portage/make.conf" \
+		 <"${files_dir}${devext_basedir}/etc/portage/make.conf.tmpl"
+
+profile=$(get_board_profile "${BOARD}")
+profile="${profile#*:}/dev"
+
+# /var/lib/portage will be overlay-mounted from ${devext_basedir} on devext initialisation
+ln -s "/var/lib/portage/coreos-overlay/profiles/${profile}" \
+        .${devext_basedir}/etc/portage/make.profile
+
+popd

build_sysext

@@ -22,14 +22,26 @@ DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build a sysext for."
 DEFINE_string metapkgs '' \
   "Comma-separated list of meta-packages to build from source and install into sysext image."
+DEFINE_string emerge_opts "" \
+  "Additional options to pass to emerge."
 DEFINE_string squashfs_base '' \
   "The path to the squashfs base image. Defaults to the most current image built in '${default_imagedir}/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}'."
 DEFINE_string image_builddir '' \
   "Custom directory to build the sysext in. Defaults to a 'sysext' sub-directory of the directory the squashfs base image resides in; '${default_imagedir}/sysext' by default."
 DEFINE_boolean strip_binaries "${FLAGS_FALSE}" \
   "After installation, scan sysext root for unstripped binaries and strip these. WARNING - this can subtly break some packages, e.g. Docker (see https://github.com/moby/moby/blob/master/project/PACKAGERS.md#stripping-binaries)."
+DEFINE_string preparefs_script '' \
+  "A path to executable that will set up the rootfs of the sysext image. Called before emerge."
+DEFINE_string preparefs_script_args '' \
+  "Additiona arguments to be passed to preparefs_script. (Arg 1 is always the image root)"
 DEFINE_string manglefs_script '' \
-  "A path to executable that will customize the rootfs of the sysext image."
+  "A path to executable that will customize the rootfs of the sysext image. Called after emerge."
+DEFINE_string manglefs_script_args '' \
+  "Additiona arguments to be passed to manglefs_script. (Arg 1 is always the image root)"
+DEFINE_boolean mangle_with_base "${FLAGS_FALSE}" \
+  "Run 'manglefs_script' with squashfs_base still merged. Makes both sysext and base files visible to mangle script."
+DEFINE_boolean remove_duplicate_files "${FLAGS_FALSE}" \
+  "Remove all files from the sysext files that are present (and identical) in squashfs_base."
 DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \
   "Generate an additional squashfs '<sysext_name>_pkginfo.raw' with portage package meta-information (/var/db ...). Useful for creating sysext dependencies; see 'base_pkginfo' below."
 DEFINE_string base_pkginfo "" \
@@ -208,10 +220,20 @@ fi
 
 if [[ -n "${FLAGS_metapkgs}" ]]; then
   mapfile -t metapkgs < <(tr ',' '\n' <<<"${FLAGS_metapkgs}")
-  "emerge-${FLAGS_board}" --nodeps --buildpkgonly --usepkg n --verbose "${metapkgs[@]}"
+  "emerge-${FLAGS_board}" --nodeps --buildpkgonly --usepkg n --verbose ${FLAGS_emerge_opts} "${metapkgs[@]}"
   set -- "${metapkgs[@]}" "${@}"
 fi
 
+export BOARD="${FLAGS_board}"
+export BUILD_DIR
+
+if [[ -n "${FLAGS_preparefs_script}" ]]; then
+  if [[ ! -x "${FLAGS_preparefs_script}" ]]; then
+    die "${FLAGS_preparefs_script} is not executable"
+  fi
+  "${FLAGS_preparefs_script}" "${BUILD_DIR}/install-root" ${FLAGS_preparefs_script_args}
+fi
+
 if [[ ${#} -lt 1 ]]; then
    error 'No packages or meta packages to install.'
    show_help_if_requested -h
@@ -219,9 +241,8 @@ fi
 
 info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression".
 
-for package; do
-  echo "Installing package into sysext image: $package"
-  FEATURES="-ebuild-locks" emerge \
+echo "Installing package(s) into sysext image: '$@'"
+FEATURES="-ebuild-locks" emerge \
     --root="${BUILD_DIR}/install-root" \
     --config-root="/build/${FLAGS_board}"  \
     --sysroot="/build/${FLAGS_board}"  \
@@ -230,14 +251,33 @@ for package; do
     --getbinpkg \
     --verbose \
     --jobs=${NUM_JOBS} \
-    "${package}"
-done
+    ${FLAGS_emerge_opts} \
+    "${@}"
+
+if [[ "${FLAGS_mangle_with_base}" == "${FLAGS_TRUE}" && -n "${FLAGS_manglefs_script}" ]]; then
+  if [[ ! -x "${FLAGS_manglefs_script}" ]]; then
+    die "${FLAGS_manglefs_script} is not executable"
+  fi
+  "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root" ${FLAGS_manglefs_script_args}
+fi
 
 # Make squashfs generation more reproducible.
 export SOURCE_DATE_EPOCH=$(stat -c '%Y' "${BUILD_DIR}/fs-root/usr/lib/os-release")
 
 # Unmount in order to get rid of the overlay
 umount "${BUILD_DIR}/install-root"
+
+# Check for duplicates before unmounting the base squashfs
+if [[ "$FLAGS_remove_duplicate_files" = "${FLAGS_TRUE}" ]] ; then
+    info "Removing duplicate (identical) files."
+    find "${BUILD_DIR}/install-root" -type f | while read sysextfile; do
+            basefile="${BUILD_DIR}/fs-root/${sysextfile#${BUILD_DIR}/install-root/}"
+            if [ -f "${basefile}" ] && cmp --silent "${sysextfile}" "${basefile}" ; then
+                rm -vf "${sysextfile}"
+            fi
+     done
+fi
+
 umount "${BUILD_DIR}/fs-root"
 
 if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then
@@ -268,11 +308,11 @@ if [[ "${FLAGS_strip_binaries}" = "${FLAGS_TRUE}" ]]; then
            done
 fi
 
-if [[ -n "${FLAGS_manglefs_script}" ]]; then
+if [[ "${FLAGS_mangle_with_base}" == "${FLAGS_FALSE}" && -n "${FLAGS_manglefs_script}" ]]; then
   if [[ ! -x "${FLAGS_manglefs_script}" ]]; then
     die "${FLAGS_manglefs_script} is not executable"
   fi
-  "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root"
+  "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root" ${FLAGS_manglefs_script_args}
 fi
 
 info "Removing non-/usr directories from sysext image"
@@ -294,6 +334,11 @@ printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-r
 
 info "Removing opaque directory markers to always merge all contents"
 find "${BUILD_DIR}/install-root" -xdev -type d -exec sh -c 'if [ "$(attr -R -q -g overlay.opaque {} 2>/dev/null)" = y ]; then attr -R -r overlay.opaque {}; fi' \;
+info "Removing white-out files so sysexts don't accidentally remove base image files "
+find "${BUILD_DIR}/install-root" -type c -exec sh -c '\
+         if [ "$(stat -c "%t.%T" {} 2>/dev/null)" = "0.0" ]; then \
+            rm -fv {}; \
+         fi' \;
 
 info "Checking for invalid file ownership"
 invalid_files=$(find "${BUILD_DIR}/install-root" -user sdk -or -group sdk)
@@ -311,4 +356,5 @@ mount -rt squashfs -o loop,nodev "${BUILD_DIR}/${SYSEXTNAME}.raw" "${BUILD_DIR}/
 write_contents "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents.txt"
 write_contents_with_technical_details "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_contents_wtd.txt"
 write_disk_space_usage_in_paths "${BUILD_DIR}/img-rootfs" "${BUILD_DIR}/${SYSEXTNAME}_disk_usage.txt"
+
 umount "${BUILD_DIR}/img-rootfs"

sdk_container/.repo/manifests/version.txt

@@ -1,4 +1,4 @@
-FLATCAR_VERSION=4117.0.0+nightly-20241008-2100
+FLATCAR_VERSION=4117.0.0+nightly-20241008-2100-1-g15242678ff
 FLATCAR_VERSION_ID=4117.0.0
-FLATCAR_BUILD_ID="nightly-20241008-2100"
+FLATCAR_BUILD_ID="nightly-20241008-2100-1-g15242678ff"
 FLATCAR_SDK_VERSION=4117.0.0+nightly-20241008-2100

sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-dev/coreos-dev-0.1.0.ebuild

@@ -18,11 +18,13 @@ KEYWORDS="amd64 arm arm64 x86"
 RDEPEND="
 	app-portage/gentoolkit
 	coreos-base/coreos
-	coreos-base/emerge-gitclone
+    dev-build/make
 	dev-debug/gdb
 	dev-debug/strace
 	dev-lang/python
 	dev-util/pahole
+	dev-util/patchelf
+	dev-util/patchutils
 	net-analyzer/netperf
 	net-analyzer/traceroute
 	net-dialup/minicom
@@ -40,9 +42,14 @@ RDEPEND="
 	sys-apps/portage
 	sys-apps/smartmontools
 	sys-apps/which
+	sys-devel/binutils
 	sys-devel/gcc
+	sys-devel/patch
 	sys-fs/lvm2
 	sys-fs/squashfs-tools
+	sys-kernel/linux-headers
+    sys-libs/readline
+    sys-libs/glibc
 	sys-process/procps
 	sys-process/psmisc
 	"

sdk_container/src/third_party/coreos-overlay/coreos/sysext/devext/usr/lib/systemd/system/init-devext.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=Flatcar developer system extension
+
+[Service]
+Type=oneshot
+ExecStart=/usr/share/flatcar/devext/devext-init.sh
+RemainAfterExit=true
+
+[Install]
+WantedBy=multi-user.target

sdk_container/src/third_party/coreos-overlay/coreos/sysext/devext/usr/lib/systemd/system/multi-user.target.d/10-devext.conf

@@ -0,0 +1,2 @@
+[Unit]
+Upholds=init-devext.service

sdk_container/src/third_party/coreos-overlay/coreos/sysext/devext/usr/lib/systemd/system/systemd-sysext.service.d/10-devext.conf

@@ -0,0 +1,6 @@
+[Service]
+ExecStart=
+ExecStart=systemd-sysext --mutable=auto refresh
+ExecReload=
+ExecReload=systemd-sysext --mutable=auto refresh
+

sdk_container/src/third_party/coreos-overlay/coreos/sysext/devext/usr/share/flatcar/devext/devext-init.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -euo pipefail
+
+version="$(source /usr/lib/extension-release.d/extension-release.devext \
+           && echo "$VERSION_ID")"
+
+basedir="/usr/share/flatcar/devext"
+vardir="/var/devext/${version}"
+
+for dir in var/db/pkg \
+           var/lib/portage \
+           etc/portage; do
+
+    workdir="${vardir}/.$(basename "${dir}")-work"
+    writedir="${vardir}/${dir}"
+    mkdir -p "/${dir}" "${writedir}" "${workdir}"
+
+    srcdir="${basedir}/${dir}"
+    mount -t overlay \
+        -o "lowerdir=${srcdir},upperdir=${writedir},workdir=${workdir}" \
+        "/usr/share/flatcar/devext/${dir}" "/${dir}"
+done
+
+# set up mutable /usr
+mkdir -p "${vardir}/usr" \
+         "/var/lib/extensions.mutable/"
+
+ln -s "${vardir}/usr" "/var/lib/extensions.mutable/"
+
+if ! touch /usr/dev-mode; then
+    systemctl reload systemd-sysext.service
+    touch /usr/dev-mode
+fi

sdk_container/src/third_party/coreos-overlay/coreos/sysext/devext/usr/share/flatcar/devext/etc/portage/make.conf.tmpl

@@ -0,0 +1,12 @@
+# make.conf for Flatcar development sysext
+ARCH=${ARCH}
+CHOST=${CHOST}
+
+# on-instance /var/lib/portage is an overlayfs backed by ${devext_basedir}/var
+DISTDIR="/var/lib/portage/distfiles"
+PKGDIR="/var/lib/portage/pkgs"
+PORT_LOGDIR="/var/log/portage"
+PORTAGE_BINHOST="${PORTAGE_BINHOST}"
+
+# Disable sandbox to keep messing with /etc to a minimum
+FEATURES="-sandbox -usersandbox"

sdk_container/src/third_party/coreos-overlay/coreos/sysext/devext/usr/share/flatcar/devext/etc/portage/repos.conf/coreos-overlay.conf

@@ -0,0 +1,2 @@
+[coreos-overlay]
+location = /var/lib/portage/coreos-overlay

sdk_container/src/third_party/coreos-overlay/coreos/sysext/devext/usr/share/flatcar/devext/etc/portage/repos.conf/portage-stable.conf

@@ -0,0 +1,5 @@
+[DEFAULT]
+main-repo = portage-stable
+
+[portage-stable]
+location = /var/lib/portage/portage-stable