Merge pull request #2694 from flatcar/setup-nvidia-fixes

nvidia.service arm64 support & fixes

changelog/bugfixes/2025-03-07-coreos-modules-cross-compilation.md

@@ -0,0 +1 @@
+- The kernel module build directory now contains native binaries in arm64 images instead of the previous amd64 binaries ([scripts#2694](https://github.com/flatcar/scripts/pull/2694))

changelog/bugfixes/2025-03-07-nvidia-driver-type.md

@@ -0,0 +1 @@
+- Nvidia driver installer service now supports the 570 driver branch by forcing the use of the proprietary kernel module. The 570 branch defaults to the kernel-open driver which requires loading firmware, which is not yet supported on Flatcar. ([scripts#2694](https://github.com/flatcar/scripts/pull/2694))

changelog/changes/2025-03-07-nvidia-arm64-support.md

@@ -0,0 +1 @@
+- Added support for ARM64 architecture in the NVIDIA driver installer service ([scripts#2694](https://github.com/flatcar/scripts/pull/2694))

changelog/updates/2025-03-07-nvidia.md

@@ -0,0 +1,2 @@
+- AMD64: nvidia-drivers ([535.230.02](https://docs.nvidia.com/datacenter/tesla/tesla-release-notes-535-230-02/index.html))
+- ARM64: nvidia-drivers ([570.86.15](https://docs.nvidia.com/datacenter/tesla/tesla-release-notes-570-86-15/index.html))

sdk_container/src/third_party/coreos-overlay/eclass/coreos-kernel.eclass

@@ -42,7 +42,7 @@ DEPEND="=sys-kernel/coreos-sources-${COREOS_SOURCE_VERSION}"
 RESTRICT="binchecks strip"
 
 # The build tools are OK and shouldn't trip up multilib-strict.
-QA_MULTILIB_PATHS="usr/lib/modules/.*/build/scripts/.*"
+QA_MULTILIB_PATHS="usr/lib/modules/.*/build/scripts/kconfig/.*"
 
 # Use source installed by coreos-sources
 # KERNEL_DIR must find the kernel source tree under /usr/src/linux-*-coreos,

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/coreos-modules-6.6.82.ebuild

@@ -54,6 +54,14 @@ src_install() {
 	# Clean up the build tree
 	shred_keys
 	kmake clean
+
+	# TODO: ensure that fixdep and kbuild tools shipped inside the image
+	# are native (we previously shipped amd64 binaries on arm64).
+	# Upstream has a new script from v6.12 that we might be able to use:
+	# scripts/package/install-extmod-build
+	kmake HOSTLD=$(tc-getLD) HOSTCC=$(tc-getCC) cmd_and_fixdep='$(cmd)' modules_prepare
+	kmake clean
+
 	find "build/" -type d -empty -delete || die
 	rm "build/.config.old" || die
 

sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/install-nvidia

@@ -4,11 +4,35 @@
 NVIDIA_DRIVER_BASENAME="$1"
 KERNEL_NAME="$(ls /lib/modules)"
 
+option_supported() {
+  local opt="$1"
+  ./nvidia-installer -A -h | grep -qe "--$opt"
+}
+
 cd "/nvidia/${NVIDIA_DRIVER_BASENAME}"
+
+EXTRA_OPTS=()
+if option_supported no-rebuild-initramfs ; then
+  EXTRA_OPTS+=( --no-rebuild-initramfs )
+fi
+if option_supported skip-module-load ; then
+  EXTRA_OPTS+=( --skip-module-load )
+fi
+
+# TODO: open requires firmware loading.
+# Can we load the module from the nspawn container?
+if option_supported kernel-module-type ; then
+  EXTRA_OPTS+=( --kernel-module-type=proprietary )
+elif option_supported kernel-module-build-directory ; then
+  EXTRA_OPTS+=( --kernel-module-build-directory=kernel )
+fi
+
 ./nvidia-installer -s -n \
   --no-check-for-alternate-installs \
   --no-kernel-module-source \
   --kernel-name="${KERNEL_NAME}" \
+  "${EXTRA_OPTS[@]}" \
+  --no-x-check \
   --no-opengl-files \
   --no-distro-scripts \
   --no-systemd \
@@ -16,6 +40,9 @@ cd "/nvidia/${NVIDIA_DRIVER_BASENAME}"
   --kernel-install-path="${PWD}/install-mod" \
   --log-file-name="${PWD}/nvidia-installer.log" || true
 
+echo "Last 50 lines of nvidia-installer.log:"
+tail -n50 nvidia-installer.log || true
+
 mkdir -p /lib/modules/${KERNEL_NAME}/video
 mkdir -p "${PWD}"/install-mod
 cp "${PWD}"/kernel/*.ko "${PWD}"/install-mod

sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/setup-nvidia

@@ -7,7 +7,7 @@ set -euo pipefail
 [ -f /etc/flatcar/nvidia-metadata ] && . /etc/flatcar/nvidia-metadata
 
 NVIDIA_DOWNLOAD_BASEURL="https://us.download.nvidia.com/${NVIDIA_PRODUCT_TYPE}/"
-NVIDIA_DRIVER_BASENAME="NVIDIA-Linux-x86_64-${NVIDIA_DRIVER_VERSION}"
+NVIDIA_DRIVER_BASENAME="NVIDIA-Linux-$(uname -m)-${NVIDIA_DRIVER_VERSION}"
 NVIDIA_WORKDIR='nvidia-workdir'
 
 FLATCAR_DEVELOPER_CONTAINER="flatcar_developer_container-${FLATCAR_RELEASE_VERSION}.bin"
@@ -38,8 +38,8 @@ function download_flatcar_developer_container() {
     fi
 
     curl -L --fail "${FLATCAR_DEVELOPER_CONTAINER_URL}" -o "${FLATCAR_ROOT_WORKDIR}/${FLATCAR_DEVELOPER_CONTAINER}.bz2"
-    lbzip2 -d "${FLATCAR_ROOT_WORKDIR}/${FLATCAR_DEVELOPER_CONTAINER}.bz2"
-
+    cp --sparse=always <(lbzcat "${FLATCAR_ROOT_WORKDIR}/${FLATCAR_DEVELOPER_CONTAINER}.bz2") "${FLATCAR_ROOT_WORKDIR}/${FLATCAR_DEVELOPER_CONTAINER}"
+    rm "${FLATCAR_ROOT_WORKDIR}/${FLATCAR_DEVELOPER_CONTAINER}.bz2"
   fi
 
   return 0
@@ -67,7 +67,7 @@ function extract_nvidia_installer() {
 function run_nspawn_container() {
   echo Spawn system-nspawn container to install the NVIDIA drivers
 
-  sudo systemd-nspawn --read-only --volatile=overlay --image="${FLATCAR_ROOT_WORKDIR}/${FLATCAR_DEVELOPER_CONTAINER}" --bind="${FLATCAR_ROOT_WORKDIR}/${NVIDIA_WORKDIR}":/nvidia --bind=/usr/lib/nvidia/bin:/app/bin/ /app/bin/install-nvidia "$NVIDIA_DRIVER_BASENAME"
+  sudo systemd-nspawn --keep-unit --register=no --read-only --volatile=overlay --image="${FLATCAR_ROOT_WORKDIR}/${FLATCAR_DEVELOPER_CONTAINER}" --overlay=/usr/lib/modules::/usr/lib/modules --bind="${FLATCAR_ROOT_WORKDIR}/${NVIDIA_WORKDIR}":/nvidia --bind=/usr/lib/nvidia/bin:/app/bin/ /app/bin/install-nvidia "$NVIDIA_DRIVER_BASENAME"
 }
 
 function copy_nvidia_build_artifacts() {
@@ -88,7 +88,11 @@ SYSEXT_LEVEL=1.0
 EOF
 
   pushd /opt/nvidia
-  ln -sfn "${NVIDIA_FLATCAR_VERSION_PAIR}" current
+  if [[ -d "${NVIDIA_CURRENT_INSTALLATION}" ]]
+  then
+    rm -rf "${NVIDIA_CURRENT_INSTALLATION}"
+  fi
+  ln -sfn -T "${NVIDIA_FLATCAR_VERSION_PAIR}" "${NVIDIA_CURRENT_INSTALLATION}"
   popd
 }
 

sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/nvidia-metadata

@@ -1,2 +1,2 @@
-NVIDIA_DRIVER_VERSION=535.216.01
+NVIDIA_DRIVER_VERSION=@PV@
 NVIDIA_PRODUCT_TYPE=tesla

sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/nvidia-drivers-535.230.02.ebuild

@@ -11,7 +11,7 @@ SRC_URI=""
 
 LICENSE="Apache-2.0"
 SLOT="0"
-KEYWORDS="amd64 arm64"
+KEYWORDS="amd64"
 IUSE=""
 
 # no source directory
@@ -23,6 +23,8 @@ src_install() {
   exeinto "/usr/lib/nvidia/bin"
   doexe "${FILESDIR}/bin/install-nvidia"
   doexe "${FILESDIR}/bin/setup-nvidia"
+  cp "${FILESDIR}/nvidia-metadata" nvidia-metadata || die "cp failed"
+  sed -i -e "s/@PV@/${PV}/" nvidia-metadata
   insinto "/usr/share/flatcar"
-  doins "${FILESDIR}/nvidia-metadata"
+  doins nvidia-metadata
 }

sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/nvidia-drivers-570.86.15.ebuild

@@ -0,0 +1,30 @@
+# Copyright (c) 2020 Kinvolk GmbH. All rights reserved.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=7
+
+inherit systemd
+
+DESCRIPTION="NVIDIA drivers"
+HOMEPAGE=""
+SRC_URI=""
+
+LICENSE="Apache-2.0"
+SLOT="0"
+KEYWORDS="arm64"
+IUSE=""
+
+# no source directory
+S="${WORKDIR}"
+
+src_install() {
+  systemd_dounit "${FILESDIR}/units/nvidia.service"
+  systemd_enable_service multi-user.target nvidia.service
+  exeinto "/usr/lib/nvidia/bin"
+  doexe "${FILESDIR}/bin/install-nvidia"
+  doexe "${FILESDIR}/bin/setup-nvidia"
+  cp "${FILESDIR}/nvidia-metadata" nvidia-metadata || die "cp failed"
+  sed -i -e "s/@PV@/${PV}/" nvidia-metadata
+  insinto "/usr/share/flatcar"
+  doins nvidia-metadata
+}