From 5e4cc8f6e966eb7a2d7c2d9e93a382bb68894dff Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Tue, 20 May 2025 14:31:43 +0100 Subject: [PATCH 1/2] sys-kernel/coreos-modules: Install external mod build files with script The kernel now includes a script for installing the files needed to build out-of-tree modules, rendering our existing code obsolete. The layout is different, but we were following Ubuntu's non-standard layout when there was no need to. Ubuntu's approach is seemingly designed to save space by symlinking common files across different platforms, but Flatcar doesn't need to do this. More importantly, our previous approach relied on a kernel patch we have carried for years that no longer applies from v6.13. The patch cannot simply be reworked as the underlying mechanism has changed. This clears the last major blocker for the arm64 SDK as the previous approach also relied on implicit execution by QEMU. There has been concern that this may break compatibility with some modules, but I have not seen any issues in practise. I have symlinked `source` to `build` even though we don't install the full kernel sources because this is what Fedora does, and it makes the layout resemble Ubuntu a little more. Should any issues arise, I will gladly work with upstreams to resolve them or otherwise make adjustments. Signed-off-by: James Le Cuirot --- changelog/changes/2025-10-30-kmod-build.md | 1 + .../eclass/coreos-kernel.eclass | 86 +------------------ .../coreos-modules-6.12.54.ebuild | 39 ++++----- 3 files changed, 18 insertions(+), 108 deletions(-) create mode 100644 changelog/changes/2025-10-30-kmod-build.md diff --git a/changelog/changes/2025-10-30-kmod-build.md b/changelog/changes/2025-10-30-kmod-build.md new file mode 100644 index 0000000000..009254d3fd --- /dev/null +++ b/changelog/changes/2025-10-30-kmod-build.md @@ -0,0 +1 @@ +- The way that files for building custom kernel modules are installed has changed from a Ubuntu-inspired method to the standard upstream kernel method. In the unlikely event that this breaks your module builds, please let the Flatcar team know immediately. diff --git a/sdk_container/src/third_party/coreos-overlay/eclass/coreos-kernel.eclass b/sdk_container/src/third_party/coreos-overlay/eclass/coreos-kernel.eclass index 73b2a8b5ca..8653f315a8 100644 --- a/sdk_container/src/third_party/coreos-overlay/eclass/coreos-kernel.eclass +++ b/sdk_container/src/third_party/coreos-overlay/eclass/coreos-kernel.eclass @@ -117,10 +117,10 @@ kmake() { if gcc-specs-pie; then kernel_cflags="-nopie -fstack-check=no ${kernel_cflags}" fi - emake "--directory=${S}/source" \ + emake "--directory=${KERNEL_DIR}" \ ARCH="${kernel_arch}" \ CROSS_COMPILE="${CHOST}-" \ - KBUILD_OUTPUT="../build" \ + KBUILD_OUTPUT="${S}/build" \ KCFLAGS="${kernel_cflags}" \ LDFLAGS="" \ "V=1" \ @@ -206,85 +206,6 @@ setup_keys() { popd } -# Populate /lib/modules/$(uname -r)/{build,source} -install_build_source() { - local kernel_arch=$(tc-arch-kernel) - local host_kernel_arch=$(tc-ninja_magic_to_arch kern "${CBUILD}") - - # NOTE: We have to get ${archabspaths} before removing symlinks under - # /usr/lib/modules. However, do not exclude "dt-bindings" for now, - # as it looks architecture-independent. - local archabspaths=($(ls -1d ${D}/usr/lib/modules/${KV_FULL}/source/scripts/dtc/include-prefixes/* \ - | grep -v dt-bindings )) - - # remove the broken symlinks referencing $ROOT - rm "${D}/usr/lib/modules/${KV_FULL}/build" || die - - # Compose list of architectures to be excluded from the kernel modules - # tree in the final image. It is an array to be used as a pattern for - # grep command below at the end of "find source/scripts" command for - # fetching kernel modules list, e.g.: - # find source/scripts -follow -print \ - # | grep -E -v -w "include-prefixes/arc|include-prefixes/xtensa" - declare -a excarchlist - local excarchstr - - for apath in "${archabspaths[@]}"; do - local arch - arch=$(basename "${apath}") - if [[ "${arch}" != "${kernel_arch}" ]]; then - excarchlist+=("include-prefixes/${arch}") - - # Do not append delimiter '|' in case of the last element. - if [[ "${apath}" != "${archabspaths[-1]}" ]]; then - excarchlist+=("|") - fi - fi - done - - # Remove every whitespace from the grep pattern string, to make pattern - # matching work well. - excarchstr=$(echo "${excarchlist[@]}" | sed -e 's/[[:space:]]*//g') - - # Install a stripped source for out-of-tree module builds (Debian-derived) - # - # NOTE: we need to exclude unsupported architectures from source/scripts, - # to prevent the final image from having unnecessary directories under - # /usr/lib/modules/${KV_FULL}/source/scripts/dtc/include-prefixes. - # The grep must run with "-w" to exclude exact patterns like either arm - # or arm64. - { - echo source/Makefile - find source/arch/${host_kernel_arch} -follow -maxdepth 1 -name 'Makefile*' -print - find source/arch/${kernel_arch} -follow -maxdepth 1 -name 'Makefile*' -print - find source/arch/${kernel_arch} -follow \( -name 'module.lds' -o -name 'Kbuild.platforms' -o -name 'Platform' \) -print - find $(find source/arch/${kernel_arch} -follow \( -name include -o -name scripts \) -follow -type d -print) -print - find source/include -follow -print - find source/scripts -follow -print | grep -E -v -w "${excarchstr}" - find build/ -print - } | cpio -pd \ - --preserve-modification-time \ - --owner=root:root \ - --dereference \ - "${D}/usr/lib/modules/${KV_FULL}" || die - # ./build/source is a symbolic link so cpio ends up creating an empty dir. - # Restore the symlink. - pushd "${D}/usr/lib/modules/${KV_FULL}" - rmdir build/source || die - ln -sr source build || die - # Symlink includes into the build directory to resemble Ubuntu's /lib/modules - # layout. This lets the Nvidia driver build when passing SYSSRC=/lib/modules/../build - # instead of requiring SYSOUT/SYSSRC. - { - find source/include -mindepth 1 -maxdepth 1 -type d - find source/arch/${kernel_arch}/include -mindepth 1 -maxdepth 1 -type d - } | while read src; do - dst="${src/source/build}" - ln -sr "${src}" "${dst}" || die - done || die - popd -} - coreos-kernel_pkg_pretend() { [[ "${MERGE_TYPE}" == binary ]] && return @@ -302,10 +223,7 @@ coreos-kernel_pkg_setup() { } coreos-kernel_src_unpack() { - # we more or less reproduce the layout in /lib/modules/$(uname -r)/ mkdir -p "${S}/build" || die - mkdir -p "${S}/source" || die - ln -s "${KERNEL_DIR}"/* "${S}/source/" || die } coreos-kernel_src_configure() { diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/coreos-modules-6.12.54.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/coreos-modules-6.12.54.ebuild index 88f3f5b596..983c2321d0 100644 --- a/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/coreos-modules-6.12.54.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/coreos-modules-6.12.54.ebuild @@ -35,41 +35,32 @@ src_compile() { } src_install() { + local build="lib/modules/${KV_FULL}/build" + # Install modules to /usr. - # Install firmware to a temporary (bogus) location. - # The linux-firmware package will be used instead. # Stripping must be done here, not portage, to preserve sigs. - kmake INSTALL_MOD_PATH="${D}/usr" \ + kmake INSTALL_MOD_PATH="${ED}/usr" \ INSTALL_MOD_STRIP="--strip-debug" \ - INSTALL_FW_PATH="${T}/fw" \ modules_install # Install to /usr/lib/debug with debug symbols intact - kmake INSTALL_MOD_PATH="${D}/usr/lib/debug/usr" \ - INSTALL_FW_PATH="${T}/fw" \ + kmake INSTALL_MOD_PATH="${ED}/usr/lib/debug/usr" \ modules_install - rm "${D}/usr/lib/debug/usr/lib/modules/${KV_FULL}/"modules.* || die - rm "${D}/usr/lib/debug/usr/lib/modules/${KV_FULL}/build" || die + rm "${ED}/usr/lib/debug/usr/lib/modules/${KV_FULL}"/{build,modules.*} || die - # Clean up the build tree - kmake clean + # Replace the broken /lib/modules/${KV_FULL}/build symlink with a copy of + # the files needed to build out-of-tree modules. + rm "${ED}/usr/${build}" || die + kmake run-command KBUILD_RUN_COMMAND="${KERNEL_DIR}/scripts/package/install-extmod-build ${ED}/usr/${build}" - # TODO: ensure that fixdep and kbuild tools shipped inside the image - # are native (we previously shipped amd64 binaries on arm64). - # Upstream has a new script from v6.12 that we might be able to use: - # scripts/package/install-extmod-build - kmake HOSTLD=$(tc-getLD) HOSTCC=$(tc-getCC) cmd_and_fixdep='$(cmd)' modules_prepare - kmake clean - - find "build/" -type d -empty -delete || die - rm "build/.config.old" || die - - # Install /lib/modules/${KV_FULL}/{build,source} - install_build_source + # Install the original config because the above doesn't. + insinto "/usr/${build}" + doins build/.config # Not strictly required but this is where we used to install the config. - dodir "/usr/boot" - local build="lib/modules/${KV_FULL}/build" dosym "../${build}/.config" "/usr/boot/config-${KV_FULL}" dosym "../${build}/.config" "/usr/boot/config" + + # Symlink "source" to "build" for compatibility. Fedora does this. + dosym build "/usr/${build}/../source" } From 51eddf8f64c64a299cae731dc691f21ee7beb835 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 22 May 2025 18:38:21 +0100 Subject: [PATCH 2/2] There is no need to set KERNEL_DIR anymore now linux-mod-r1 is fixed Signed-off-by: James Le Cuirot --- .../coreos-overlay/coreos/config/env/sys-fs/zfs-kmod | 3 --- .../coreos/config/env/x11-drivers/nvidia-drivers | 3 --- 2 files changed, 6 deletions(-) diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/zfs-kmod b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/zfs-kmod index 5f7fa2a9a8..f7951a4d39 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/zfs-kmod +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/zfs-kmod @@ -1,6 +1,3 @@ -: ${MODULES_ROOT:=$(echo ${SYSROOT}/lib/modules/*)} -KERNEL_DIR="${MODULES_ROOT}/build" - # This addresses an issue with the kernel version compatibility check # when installing zfs-kmod to /build/ (e.g. via build_packages) # from its binpkg (i.e. not recompiling it). diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/x11-drivers/nvidia-drivers b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/x11-drivers/nvidia-drivers index 5f7fa2a9a8..f7951a4d39 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/x11-drivers/nvidia-drivers +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/x11-drivers/nvidia-drivers @@ -1,6 +1,3 @@ -: ${MODULES_ROOT:=$(echo ${SYSROOT}/lib/modules/*)} -KERNEL_DIR="${MODULES_ROOT}/build" - # This addresses an issue with the kernel version compatibility check # when installing zfs-kmod to /build/ (e.g. via build_packages) # from its binpkg (i.e. not recompiling it).