From e17355e6ae3d654403c88a1e9d0a6b1d80ed958c Mon Sep 17 00:00:00 2001 From: Jared Baur Date: Tue, 26 Mar 2024 15:47:34 -0700 Subject: [PATCH 1/2] Use driverLink where possible Instead of hardcoding `/run/opengl-driver`, use `addOpenGLRunpath.driverLink`. --- modules/default.nix | 5 +++-- pkgs/l4t/default.nix | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/default.nix b/modules/default.nix index 94f76eb..7186782 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -140,7 +140,7 @@ in # l4t-core provides - among others - libnvrm_gpu.so and libnvrm_mem.so. # The l4t-core/lib directory is directly set in the DT_RUNPATH of # l4t-cuda's libcuda.so, thus the standard driver doesn't need them to be - # added in /run/opengl-driver. + # added in ${driverLink}. # # However, this isn't the case for cuda_compat's driver currently, which # is why we're including this derivation in extraPackages. @@ -269,6 +269,7 @@ in }; # Used by libEGL_nvidia.so.0 - environment.etc."egl/egl_external_platform.d".source = "/run/opengl-driver/share/egl/egl_external_platform.d/"; + environment.etc."egl/egl_external_platform.d".source = + "${pkgs.addOpenGLRunpath.driverLink}/share/egl/egl_external_platform.d/"; }; } diff --git a/pkgs/l4t/default.nix b/pkgs/l4t/default.nix index cb80aa6..9b885fa 100644 --- a/pkgs/l4t/default.nix +++ b/pkgs/l4t/default.nix @@ -1,5 +1,6 @@ { stdenv , stdenvNoCC +, addOpenGLRunpath , lib , fetchurl , fetchpatch @@ -94,7 +95,7 @@ let # l4t-core. Unfortunately, calling dlopen from libnvos.so instead of the # original library/executable means that dlopen will use the DT_RUNPATH # from libnvos.so instead of the binary/library which called it. We - # typically just need /run/opengl-driver/lib anyway, so lets add it to + # typically just need ${driverLink}/lib anyway, so lets add it to # libnvos.so here instead. # # We append a postFixupHook since we need to have this happen after @@ -102,7 +103,7 @@ let # TODO: Use runtimeDependencies instead preFixup = '' postFixupHooks+=(' - patchelf --add-rpath /run/opengl-driver/lib $out/lib/libnvos.so + patchelf --add-rpath ${addOpenGLRunpath.driverLink}/lib $out/lib/libnvos.so ') ''; }; From 12d4ee754896b53c11a8e1385ba2e2071ae4f507 Mon Sep 17 00:00:00 2001 From: Jared Baur Date: Tue, 26 Mar 2024 15:49:54 -0700 Subject: [PATCH 2/2] Update nvidia-ctk and use RUNTIME_DIRECTORY when calling nvidia-ctk Updates nvidia-ctk since some patches were applied upstream. Also changes the systemd service to use RUNTIME_DIRECTORY (this is what upstream nixpkgs does when they call nvidia-ctk). --- modules/default.nix | 32 +++++++++++++++----------------- pkgs/containers/nvidia-ctk.nix | 33 ++++++++++++++------------------- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/modules/default.nix b/modules/default.nix index 7186782..0fe72be 100644 --- a/modules/default.nix +++ b/modules/default.nix @@ -242,30 +242,28 @@ in otaUtils # Tools for UEFI capsule updates ]; - systemd.tmpfiles.rules = lib.optional nvidiaContainerRuntimeActive "d /var/run/cdi 0755 root root - -"; - systemd.services.nvidia-cdi-generate = { enable = nvidiaContainerRuntimeActive; serviceConfig = { Type = "oneshot"; RemainAfterExit = true; - ExecStart = - let - exe = "${pkgs.nvidia-jetpack.nvidia-ctk}/bin/nvidia-ctk"; - in - toString [ - exe - "cdi" - "generate" - "--nvidia-ctk-path=${exe}" # it is odd that this is needed, should be the same as /proc/self/exe? - "--driver-root=${pkgs.nvidia-jetpack.containerDeps}" # the root where nvidia libs will be resolved from - "--dev-root=/" # the root where chardevs will be resolved from - "--mode=csv" - "--csv.file=${pkgs.nvidia-jetpack.l4tCsv}" - "--output=/var/run/cdi/jetpack-nixos" # a yaml file extension is added by the nvidia-ctk tool - ]; + RuntimeDirectory = "cdi"; }; wantedBy = [ "multi-user.target" ]; + script = + let + exe = lib.getExe pkgs.nvidia-jetpack.nvidia-ctk; + in + '' + ${exe} cdi generate \ + --nvidia-ctk-path=${exe} \ + --driver-root=${pkgs.nvidia-jetpack.containerDeps} \ + --ldconfig-path ${lib.getExe' pkgs.glibc "ldconfig"} \ + --dev-root=/ \ + --mode=csv \ + --csv.file=${pkgs.nvidia-jetpack.l4tCsv} \ + --output="$RUNTIME_DIRECTORY/jetpack-nixos" + ''; }; # Used by libEGL_nvidia.so.0 diff --git a/pkgs/containers/nvidia-ctk.nix b/pkgs/containers/nvidia-ctk.nix index a08e98a..7451e1b 100644 --- a/pkgs/containers/nvidia-ctk.nix +++ b/pkgs/containers/nvidia-ctk.nix @@ -1,39 +1,32 @@ -{ fetchFromGitLab, buildGoModule, fetchpatch }: +{ buildGoModule, fetchFromGitHub, fetchpatch }: buildGoModule rec { pname = "nvidia-ctk"; - version = "unstable-${builtins.substring 0 7 src.rev}"; + version = "1.15.0-rc.4"; # TODO(jared): pin to v1.15.0 once it is released # We currently rely on some features in an unreleased version of nvidia # container toolkit. - src = fetchFromGitLab { + src = fetchFromGitHub { owner = "nvidia"; - repo = "container-toolkit/container-toolkit"; - rev = "a2262d00cc6d98ac2e95ae2f439e699a7d64dc17"; - hash = "sha256-Oi04PIES0qTih/EiFBStIoBadM3H52+81KEfUumQcIs="; + repo = "nvidia-container-toolkit"; + rev = "v${version}"; + hash = "sha256-Ky0mGothIq5BOAHc4ujrMrh1niBYUoSgaRnv30ymjsE="; }; patches = [ # ensure nvidia-ctk can build with Go versions less than 1.20 (currently # required on their latest release) (fetchpatch { - name = "Fix-double-error-wrap-fmt"; - url = "https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/commit/80756d00a6b75761103c50f605cece5fa7e39392.patch"; - hash = "sha256-hoeMUUPWKToCR7V/JG26wF6SCoHQwQORcGimH6EXDJ8="; - }) - (fetchpatch { - name = "Use-golang-1.17"; - url = "https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/commit/5956b04096d1a92b241b13cc1f3e208f8b99eea0.patch"; - hash = "sha256-VB3+ijc2Pdlm1W2LqvCjx9KDYKinWBkr/eiUJEwig/o="; + name = "Add-errors-Join-wrapper"; + url = "https://github.com/NVIDIA/nvidia-container-toolkit/commit/92f17e94939bf8c213419749f5f7b48d2f0e618c.patch"; + hash = "sha256-ioWstYky7LbIGtlfMMlbhIVN8yH7Qgp3z4wrkytT3TY="; }) (fetchpatch { - name = "Draft-Compat-with-golang-1.17"; - url = "https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/commit/86f68a49014a4cffb7dcb51f14a02f6f1816b2ee.patch"; - hash = "sha256-ioWstYky7LbIGtlfMMlbhIVN8yH7Qgp3z4wrkytT3TY="; + name = "Fix-double-error-wrap-fmt"; + url = "https://github.com/NVIDIA/nvidia-container-toolkit/commit/f23fd2ce38ee3a9e87ac41c265b637cf97990ac7.patch"; + hash = "sha256-hoeMUUPWKToCR7V/JG26wF6SCoHQwQORcGimH6EXDJ8="; }) - # ensure nvidia-ctk can find ldconfig - ./nixos-ldconfig.patch ]; subPackages = [ "cmd/nvidia-ctk" ]; @@ -41,4 +34,6 @@ buildGoModule rec { vendorHash = null; ldflags = [ "-s" "-w" "-extldflags=-Wl,-z,lazy" ]; + + meta.mainProgram = "nvidia-ctk"; }