From a6c4454b360823f788513ceb27cf0a2ab92d674a Mon Sep 17 00:00:00 2001
From: Jeremi Piotrowski <jpiotrowski@microsoft.com>
Date: Mon, 25 Apr 2022 14:00:09 +0200
Subject: [PATCH] x11-drivers/nvidia-drivers: rework install paths of files to
 match OS

This involves putting libraries under /usr/lib64 and kernel modules under
/usr/lib/module. This is an experiment at making the nvidia installation work
as a sysext as well, but there are still some issues around that. The major
issue was that `systemd-sysext refresh` would remove the OEM symlink and I
don't feel comfortable with `systemctl restart systemd-sysext` from within
another unit.

If anyone wants to try it, it's now a matter of:

  ln -s /opt/nvidia/current /run/extensions/nvidia-driver

Bonus points for moving nvidia binaries from /opt/bin to
/opt/nvidia/current/usr/bin.
---
 .../nvidia-drivers/files/bin/install-nvidia   |  9 +++++-
 .../nvidia-drivers/files/bin/setup-nvidia     | 32 +++++++++++++------
 2 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/install-nvidia b/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/install-nvidia
index ffc2903cea..803934b190 100644
--- a/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/install-nvidia
+++ b/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/install-nvidia
@@ -13,5 +13,12 @@ cd "/nvidia/${NVIDIA_DRIVER_BASENAME}"
   --no-distro-scripts \
   --no-systemd \
   --skip-depmod \
-  --kernel-install-path="${PWD}" \
+  --kernel-install-path="${PWD}/install-mod" \
   --log-file-name="${PWD}/nvidia-installer.log" || true
+
+mkdir -p /lib/modules/${KERNEL_NAME}/video
+mkdir -p "${PWD}"/install-mod
+cp "${PWD}"/kernel/*.ko "${PWD}"/install-mod
+cp "${PWD}"/install-mod/*.ko /lib/modules/${KERNEL_NAME}/video/
+depmod -a
+cp /lib/modules/${KERNEL_NAME}/modules.* "${PWD}/install-mod/"
diff --git a/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/setup-nvidia b/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/setup-nvidia
index 8e48f52cde..0838357c25 100644
--- a/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/setup-nvidia
+++ b/sdk_container/src/third_party/coreos-overlay/x11-drivers/nvidia-drivers/files/bin/setup-nvidia
@@ -74,14 +74,21 @@ function run_nspawn_container() {
 }
 
 function copy_nvidia_build_artifacts() {
-  mkdir -p "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/lib64"
-  cp "$FLATCAR_ROOT_WORKDIR/$NVIDIA_WORKDIR/${NVIDIA_DRIVER_BASENAME}"/*.so.* "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/lib64/"
+  mkdir -p "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/usr/lib64"
+  cp "$FLATCAR_ROOT_WORKDIR/$NVIDIA_WORKDIR/${NVIDIA_DRIVER_BASENAME}"/*.so.* "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/usr/lib64/"
 
   mkdir -p /opt/bin
   cp "$FLATCAR_ROOT_WORKDIR/$NVIDIA_WORKDIR/${NVIDIA_DRIVER_BASENAME}"/{nvidia-debugdump,nvidia-cuda-mps-control,nvidia-xconfig,nvidia-modprobe,nvidia-smi,nvidia-cuda-mps-server,nvidia-persistenced,nvidia-settings} /opt/bin/
 
-  mkdir -p "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/lib64/modules/$(uname -r)/video/"
-  cp "$FLATCAR_ROOT_WORKDIR/$NVIDIA_WORKDIR/${NVIDIA_DRIVER_BASENAME}"/kernel/*.ko "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/lib64/modules/$(uname -r)/video/"
+  mkdir -p "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/usr/lib/modules/$(uname -r)/video/"
+  cp "$FLATCAR_ROOT_WORKDIR/$NVIDIA_WORKDIR/${NVIDIA_DRIVER_BASENAME}"/install-mod/*.ko "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/usr/lib/modules/$(uname -r)/video/"
+  cp "$FLATCAR_ROOT_WORKDIR/$NVIDIA_WORKDIR/${NVIDIA_DRIVER_BASENAME}"/install-mod/modules.* "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/usr/lib/modules/$(uname -r)/"
+
+  mkdir -p "/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/usr/lib/extension-release.d/"
+  cat <<EOF >"/opt/nvidia/${NVIDIA_FLATCAR_VERSION_PAIR}/usr/lib/extension-release.d/extension-release.nvidia-driver"
+ID=flatcar
+SYSEXT_LEVEL=1.0
+EOF
 
   pushd /opt/nvidia
   ln -sfn "${NVIDIA_FLATCAR_VERSION_PAIR}" current
@@ -89,10 +96,18 @@ function copy_nvidia_build_artifacts() {
 }
 
 function install_and_load() {
+  # This creates symlinks to sonames
+  mkdir -p /etc/ld.so.conf.d/
+  echo "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/usr/lib64" > /etc/ld.so.conf.d/nvidia.conf
+  ldconfig
+
   modprobe -a i2c_core ipmi_msghandler ipmi_devintf
-  insmod "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/lib64/modules/$(uname -r)/video/nvidia.ko"
-  insmod "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/lib64/modules/$(uname -r)/video/nvidia-modeset.ko"
-  insmod "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/lib64/modules/$(uname -r)/video/nvidia-uvm.ko"
+
+  pushd "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/usr/lib/modules/$(uname -r)/video/"
+  insmod nvidia.ko
+  insmod nvidia-modeset.ko
+  insmod nvidia-uvm.ko
+  popd
 
   if [ ! -c /dev/nvidiactl ]
   then
@@ -104,9 +119,6 @@ function install_and_load() {
     mknod -m 666 /dev/nvidia0 c 195 0
   fi
 
-  mkdir -p /etc/ld.so.conf.d/
-  echo "/opt/nvidia/${NVIDIA_CURRENT_INSTALLATION}/lib64" > /etc/ld.so.conf.d/nvidia.conf
-  ldconfig
 }
 
 function verify_installation() {