From b84913e25f02ce3ebfeff84a2292fa3186233ab6 Mon Sep 17 00:00:00 2001 From: Ikey Doherty Date: Sun, 28 Jan 2018 19:31:35 +0000 Subject: [PATCH 1/4] snap-confine/nvidia: Support legacy biarch trees for GLVND systems GLVND systems are well catered for on biarch configurations, however the older 304 + 340 series will rely on ld.so.conf tricks to mask the system libGL.so with one in the private `libdir/nvidia*/` files. This is seen consistently across Arch, Fedora and Solus. As such we allow these fallback paths to work so that users of snapd on glvnd enabled distros with the older legacy drivers can still enjoy snapd + NVIDIA together. Signed-off-by: Ikey Doherty --- cmd/snap-confine/mount-support-nvidia.c | 16 ++++++++++++++++ interfaces/builtin/opengl.go | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cmd/snap-confine/mount-support-nvidia.c b/cmd/snap-confine/mount-support-nvidia.c index 42f56c3dce6..35913d11157 100644 --- a/cmd/snap-confine/mount-support-nvidia.c +++ b/cmd/snap-confine/mount-support-nvidia.c @@ -96,6 +96,10 @@ static const char *nvidia_globs[] = { "/usr/lib/libnvidia-ptxjitcompiler.so*", "/usr/lib/libnvidia-tls.so*", "/usr/lib/vdpau/libvdpau_nvidia.so*", + "/usr/lib/nvidia*/libGL.so*", + "/usr/lib/nvidia*/libEGL.so*", + "/usr/lib/nvidia*/libGLESv1_CM.so*", + "/usr/lib/nvidia*/libGLESv2.so*", }; static const size_t nvidia_globs_len = @@ -133,6 +137,10 @@ static const char *nvidia_globs32[] = { "/usr/lib32/libnvidia-ptxjitcompiler.so*", "/usr/lib32/libnvidia-tls.so*", "/usr/lib32/vdpau/libvdpau_nvidia.so*", + "/usr/lib32/nvidia*/libGL.so*", + "/usr/lib32/nvidia*/libEGL.so*", + "/usr/lib32/nvidia*/libGLESv1_CM.so*", + "/usr/lib32/nvidia*/libGLESv2.so*", }; static const size_t nvidia_globs32_len = @@ -216,6 +224,14 @@ static void sc_populate_libgl_with_hostfs_symlinks(const char *libgl_dir, "%s/%s", libgl_dir, filename); debug("creating symbolic link %s -> %s", symlink_name, symlink_target); + + // Make sure we don't have some link already (merged GLVND systems) + if (lstat(symlink_name, &stat_buf) == 0) { + if (unlink(symlink_name) != 0) { + die("cannot remove symbolic link target %s", symlink_name); + } + } + if (symlink(symlink_target, symlink_name) != 0) { die("cannot create symbolic link %s -> %s", symlink_name, symlink_target); diff --git a/interfaces/builtin/opengl.go b/interfaces/builtin/opengl.go index bd65881579d..3625835277f 100644 --- a/interfaces/builtin/opengl.go +++ b/interfaces/builtin/opengl.go @@ -48,7 +48,7 @@ const openglConnectedPlugAppArmor = ` /var/lib/snapd/hostfs/usr/share/vulkan/icd.d/*nvidia*.json r, # Main bi-arch GL libraries -/var/lib/snapd/hostfs/{,usr/}lib{,32,64,x32}/{,@{multiarch}/}lib{GL,EGL,GLX}.so{,.*} rm, +/var/lib/snapd/hostfs/{,usr/}lib{,32,64,x32}/{,@{multiarch}/}{,nvidia*/}lib{GL,EGL,GLX}.so{,.*} rm, /dev/dri/ r, /dev/dri/card0 rw, From 17e800f064eaa4460cc7e5ffc004056f1acad8e3 Mon Sep 17 00:00:00 2001 From: Ikey Doherty Date: Tue, 30 Jan 2018 11:06:02 +0000 Subject: [PATCH 2/4] snap-confine/nvidia: Implement robust approach to NVIDIA files Instead of our two hard-coded path lists, we'll form relative paths for all optional elements in the list against common ancestors known for biarch distributions. This builds upon the previous work and allows us to replace or grab files from the subdirectories, and also allows the private NVIDIA libraries to exist either within the toplevel libdir or within a private directory, as seen on Fedora. The flow is changed slightly so that our first primary call for each libdir is the only mount, and then we follow up with a simple populate call for the libdir. Signed-off-by: Ikey Doherty --- cmd/snap-confine/mount-support-nvidia.c | 182 +++++++++++++----------- 1 file changed, 95 insertions(+), 87 deletions(-) diff --git a/cmd/snap-confine/mount-support-nvidia.c b/cmd/snap-confine/mount-support-nvidia.c index 35913d11157..74513c1b6e2 100644 --- a/cmd/snap-confine/mount-support-nvidia.c +++ b/cmd/snap-confine/mount-support-nvidia.c @@ -42,9 +42,11 @@ #define SC_LIBGL32_DIR "/var/lib/snapd/lib/gl32" #define SC_VULKAN_DIR "/var/lib/snapd/lib/vulkan" +#define SC_VULKAN_SOURCE_DIR "/usr/share/vulkan" + // Location for NVIDIA vulkan files (including _wayland) static const char *vulkan_globs[] = { - "/usr/share/vulkan/icd.d/*nvidia*.json", + "icd.d/*nvidia*.json", }; static const size_t vulkan_globs_len = @@ -65,87 +67,42 @@ static const size_t vulkan_globs_len = // FIXME: this doesn't yet work with libGLX and libglvnd redirector // FIXME: this still doesn't work with the 361 driver static const char *nvidia_globs[] = { - "/usr/lib/libEGL.so*", - "/usr/lib/libEGL_nvidia.so*", - "/usr/lib/libGL.so*", - "/usr/lib/libOpenGL.so*", - "/usr/lib/libGLESv1_CM.so*", - "/usr/lib/libGLESv1_CM_nvidia.so*", - "/usr/lib/libGLESv2.so*", - "/usr/lib/libGLESv2_nvidia.so*", - "/usr/lib/libGLX_indirect.so*", - "/usr/lib/libGLX_nvidia.so*", - "/usr/lib/libGLX.so*", - "/usr/lib/libGLdispatch.so*", - "/usr/lib/libGLU.so*", - "/usr/lib/libXvMCNVIDIA.so*", - "/usr/lib/libXvMCNVIDIA_dynamic.so*", - "/usr/lib/libcuda.so*", - "/usr/lib/libnvcuvid.so*", - "/usr/lib/libnvidia-cfg.so*", - "/usr/lib/libnvidia-compiler.so*", - "/usr/lib/libnvidia-eglcore.so*", - "/usr/lib/libnvidia-egl-wayland*", - "/usr/lib/libnvidia-encode.so*", - "/usr/lib/libnvidia-fatbinaryloader.so*", - "/usr/lib/libnvidia-fbc.so*", - "/usr/lib/libnvidia-glcore.so*", - "/usr/lib/libnvidia-glsi.so*", - "/usr/lib/libnvidia-ifr.so*", - "/usr/lib/libnvidia-ml.so*", - "/usr/lib/libnvidia-ptxjitcompiler.so*", - "/usr/lib/libnvidia-tls.so*", - "/usr/lib/vdpau/libvdpau_nvidia.so*", - "/usr/lib/nvidia*/libGL.so*", - "/usr/lib/nvidia*/libEGL.so*", - "/usr/lib/nvidia*/libGLESv1_CM.so*", - "/usr/lib/nvidia*/libGLESv2.so*", + "libEGL.so*", + "libEGL_nvidia.so*", + "libGL.so*", + "libOpenGL.so*", + "libGLESv1_CM.so*", + "libGLESv1_CM_nvidia.so*", + "libGLESv2.so*", + "libGLESv2_nvidia.so*", + "libGLX_indirect.so*", + "libGLX_nvidia.so*", + "libGLX.so*", + "libGLdispatch.so*", + "libGLU.so*", + "libXvMCNVIDIA.so*", + "libXvMCNVIDIA_dynamic.so*", + "libcuda.so*", + "libnvcuvid.so*", + "libnvidia-cfg.so*", + "libnvidia-compiler.so*", + "libnvidia-eglcore.so*", + "libnvidia-egl-wayland*", + "libnvidia-encode.so*", + "libnvidia-fatbinaryloader.so*", + "libnvidia-fbc.so*", + "libnvidia-glcore.so*", + "libnvidia-glsi.so*", + "libnvidia-ifr.so*", + "libnvidia-ml.so*", + "libnvidia-ptxjitcompiler.so*", + "libnvidia-tls.so*", + "vdpau/libvdpau_nvidia.so*", }; static const size_t nvidia_globs_len = sizeof nvidia_globs / sizeof *nvidia_globs; -// 32-bit variants of the NVIDIA driver libraries -static const char *nvidia_globs32[] = { - "/usr/lib32/libEGL.so*", - "/usr/lib32/libEGL_nvidia.so*", - "/usr/lib32/libGL.so*", - "/usr/lib32/libOpenGL.so*", - "/usr/lib32/libGLESv1_CM.so*", - "/usr/lib32/libGLESv1_CM_nvidia.so*", - "/usr/lib32/libGLESv2.so*", - "/usr/lib32/libGLESv2_nvidia.so*", - "/usr/lib32/libGLX_indirect.so*", - "/usr/lib32/libGLX_nvidia.so*", - "/usr/lib32/libGLX.so*", - "/usr/lib32/libGLdispatch.so*", - "/usr/lib32/libGLU.so*", - "/usr/lib32/libXvMCNVIDIA.so*", - "/usr/lib32/libXvMCNVIDIA_dynamic.so*", - "/usr/lib32/libcuda.so*", - "/usr/lib32/libnvcuvid.so*", - "/usr/lib32/libnvidia-cfg.so*", - "/usr/lib32/libnvidia-compiler.so*", - "/usr/lib32/libnvidia-eglcore.so*", - "/usr/lib32/libnvidia-encode.so*", - "/usr/lib32/libnvidia-fatbinaryloader.so*", - "/usr/lib32/libnvidia-fbc.so*", - "/usr/lib32/libnvidia-glcore.so*", - "/usr/lib32/libnvidia-glsi.so*", - "/usr/lib32/libnvidia-ifr.so*", - "/usr/lib32/libnvidia-ml.so*", - "/usr/lib32/libnvidia-ptxjitcompiler.so*", - "/usr/lib32/libnvidia-tls.so*", - "/usr/lib32/vdpau/libvdpau_nvidia.so*", - "/usr/lib32/nvidia*/libGL.so*", - "/usr/lib32/nvidia*/libEGL.so*", - "/usr/lib32/nvidia*/libGLESv1_CM.so*", - "/usr/lib32/nvidia*/libGLESv2.so*", -}; - -static const size_t nvidia_globs32_len = - sizeof nvidia_globs32 / sizeof *nvidia_globs32; - #endif // ifdef NVIDIA_BIARCH // Populate libgl_dir with a symlink farm to files matching glob_list. @@ -155,7 +112,11 @@ static const size_t nvidia_globs32_len = // If the library is a symbolic link then relative links are kept as-is but // absolute links are translated to have "/path/to/hostfs" up front so that // they work after the pivot_root elsewhere. +// +// The glob list passed to us is produced with paths relative to source dir, +// to simplify the various tie-in points with this function. static void sc_populate_libgl_with_hostfs_symlinks(const char *libgl_dir, + const char *source_dir, const char *glob_list[], size_t glob_list_len) { @@ -164,13 +125,17 @@ static void sc_populate_libgl_with_hostfs_symlinks(const char *libgl_dir, // Find all the entries matching the list of globs for (size_t i = 0; i < glob_list_len; ++i) { const char *glob_pattern = glob_list[i]; - int err = - glob(glob_pattern, i ? GLOB_APPEND : 0, NULL, &glob_res); + char glob_pattern_full[512] = { 0 }; + sc_must_snprintf(glob_pattern_full, sizeof glob_pattern_full, + "%s/%s", source_dir, glob_pattern); + + int err = glob(glob_pattern_full, i ? GLOB_APPEND : 0, NULL, + &glob_res); // Not all of the files have to be there (they differ depending on the // driver version used). Ignore all errors that are not GLOB_NOMATCH. if (err != 0 && err != GLOB_NOMATCH) { die("cannot search using glob pattern %s: %d", - glob_pattern, err); + glob_pattern_full, err); } } // Symlink each file found @@ -228,7 +193,8 @@ static void sc_populate_libgl_with_hostfs_symlinks(const char *libgl_dir, // Make sure we don't have some link already (merged GLVND systems) if (lstat(symlink_name, &stat_buf) == 0) { if (unlink(symlink_name) != 0) { - die("cannot remove symbolic link target %s", symlink_name); + die("cannot remove symbolic link target %s", + symlink_name); } } @@ -240,6 +206,7 @@ static void sc_populate_libgl_with_hostfs_symlinks(const char *libgl_dir, } static void sc_mkdir_and_mount_and_glob_files(const char *rootfs_dir, + const char *source_dir, const char *tgt_dir, const char *glob_list[], size_t glob_list_len) @@ -259,7 +226,7 @@ static void sc_mkdir_and_mount_and_glob_files(const char *rootfs_dir, die("cannot mount tmpfs at %s", libgl_dir); }; // Populate libgl_dir with symlinks to libraries from hostfs - sc_populate_libgl_with_hostfs_symlinks(libgl_dir, glob_list, + sc_populate_libgl_with_hostfs_symlinks(libgl_dir, source_dir, glob_list, glob_list_len); // Remount $tgt_dir (i.e. .../lib/gl) read only debug("remounting tmpfs as read-only %s", libgl_dir); @@ -270,12 +237,52 @@ static void sc_mkdir_and_mount_and_glob_files(const char *rootfs_dir, #ifdef NVIDIA_BIARCH +// Copy the symlink farm to an already mounted/constructed target +static void sc_glob_files_only(const char *rootfs_dir, + const char *source_dir, + const char *tgt_dir, + const char *glob_list[], size_t glob_list_len) +{ + char buf[512] = { 0 }; + sc_must_snprintf(buf, sizeof(buf), "%s%s", rootfs_dir, tgt_dir); + const char *libgl_dir = buf; + + // Populate libgl_dir with symlinks to libraries from hostfs + sc_populate_libgl_with_hostfs_symlinks(libgl_dir, source_dir, glob_list, + glob_list_len); +} + +// Expose host NVIDIA drivers to the snap on biarch systems. +// +// Order is absolutely imperative here. We'll attempt to find the +// primary files for the architecture in the main directory, and end +// up copying any files across. However it is possible we're using a +// GLVND enabled host, in which case we copied libGL* to the farm. +// The next step in the list is to look within the private nvidia +// directory, exposed using ld.so.conf tricks within the host OS. +// In some distros (i.e. Solus) only the private libGL/libEGL files +// may be found here, and they'll clobber the existing GLVND files from +// the previous run. +// In other distros (like Fedora) all NVIDIA libraries are contained +// within the private directory, so we clobber the GLVND files and we +// also grab all the private NVIDIA libraries. +// +// In non GLVND cases we just copy across the exposed libGLs and NVIDIA +// libraries from wherever we find, and clobbering is also harmless. static void sc_mount_nvidia_driver_biarch(const char *rootfs_dir) { - sc_mkdir_and_mount_and_glob_files(rootfs_dir, SC_LIBGL_DIR, + // Primary arch + sc_mkdir_and_mount_and_glob_files(rootfs_dir, "/usr/lib", SC_LIBGL_DIR, nvidia_globs, nvidia_globs_len); - sc_mkdir_and_mount_and_glob_files(rootfs_dir, SC_LIBGL32_DIR, - nvidia_globs32, nvidia_globs32_len); + sc_glob_files_only(rootfs_dir, "/usr/lib/nvidia*", + SC_LIBGL_DIR, nvidia_globs, nvidia_globs_len); + + // Alternative 32-bit support + sc_mkdir_and_mount_and_glob_files(rootfs_dir, "/usr/lib32", + SC_LIBGL32_DIR, nvidia_globs, + nvidia_globs_len); + sc_glob_files_only(rootfs_dir, "/usr/lib32/nvidia*", + SC_LIBGL32_DIR, nvidia_globs, nvidia_globs_len); } #endif // ifdef NVIDIA_BIARCH @@ -377,6 +384,7 @@ void sc_mount_nvidia_driver(const char *rootfs_dir) #endif // ifdef NVIDIA_BIARCH // Common for both driver mechanisms - sc_mkdir_and_mount_and_glob_files(rootfs_dir, SC_VULKAN_DIR, - vulkan_globs, vulkan_globs_len); + sc_mkdir_and_mount_and_glob_files(rootfs_dir, SC_VULKAN_SOURCE_DIR, + SC_VULKAN_DIR, vulkan_globs, + vulkan_globs_len); } From c724c06a6a58e64c0a701cc78f9f5154164abe32 Mon Sep 17 00:00:00 2001 From: Ikey Doherty Date: Tue, 30 Jan 2018 17:20:52 +0000 Subject: [PATCH 3/4] snap-confine/nvidia: Allow configuring the libdir + lib32dir This will allow each distro to provide the native arch and the alternative lib32 directory without us trying to guess where they go. With this change, we'll only try to mount the alternative 32-bit biarch space if we're a 64-bit build of snapd, otherwise we'd only have the native 32-bit arch to worry about in the first place. Signed-off-by: Ikey Doherty --- cmd/Makefile.am | 2 +- cmd/configure.ac | 7 +++++++ cmd/snap-confine/mount-support-nvidia.c | 16 ++++++++++------ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/Makefile.am b/cmd/Makefile.am index 505b75aa554..20caac96f93 100644 --- a/cmd/Makefile.am +++ b/cmd/Makefile.am @@ -205,7 +205,7 @@ snap_confine_snap_confine_SOURCES = \ snap-confine/user-support.c \ snap-confine/user-support.h -snap_confine_snap_confine_CFLAGS = $(CHECK_CFLAGS) $(AM_CFLAGS) -DLIBEXECDIR=\"$(libexecdir)\" +snap_confine_snap_confine_CFLAGS = $(CHECK_CFLAGS) $(AM_CFLAGS) -DLIBEXECDIR=\"$(libexecdir)\" -DNATIVE_LIBDIR=\"$(libdir)\" snap_confine_snap_confine_LDFLAGS = $(AM_LDFLAGS) snap_confine_snap_confine_LDADD = libsnap-confine-private.a snap_confine_snap_confine_CFLAGS += $(LIBUDEV_CFLAGS) diff --git a/cmd/configure.ac b/cmd/configure.ac index f2e6ce6efbe..74f74743793 100644 --- a/cmd/configure.ac +++ b/cmd/configure.ac @@ -209,5 +209,12 @@ AC_ARG_ENABLE([static-libseccomp], esac], [enable_static_libseccomp=no]) AM_CONDITIONAL([STATIC_LIBSECCOMP], [test "x$enable_static_libseccomp" = "xyes"]) +LIB32_DIR="${prefix}/lib32" +AC_ARG_WITH([32bit-libdir], + AS_HELP_STRING([--with-32bit-libdir=DIR], [Use an alternate lib32 directory]), + [LIB32_DIR="$withval"]) +AC_SUBST(LIB32_DIR) +AC_DEFINE_UNQUOTED([LIB32_DIR], "${LIB32_DIR}", [Location of the lib32 directory]) + AC_CONFIG_FILES([Makefile]) AC_OUTPUT diff --git a/cmd/snap-confine/mount-support-nvidia.c b/cmd/snap-confine/mount-support-nvidia.c index 74513c1b6e2..e7ddf0a961f 100644 --- a/cmd/snap-confine/mount-support-nvidia.c +++ b/cmd/snap-confine/mount-support-nvidia.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "../libsnap-confine-private/classic.h" @@ -272,17 +273,20 @@ static void sc_glob_files_only(const char *rootfs_dir, static void sc_mount_nvidia_driver_biarch(const char *rootfs_dir) { // Primary arch - sc_mkdir_and_mount_and_glob_files(rootfs_dir, "/usr/lib", SC_LIBGL_DIR, - nvidia_globs, nvidia_globs_len); - sc_glob_files_only(rootfs_dir, "/usr/lib/nvidia*", - SC_LIBGL_DIR, nvidia_globs, nvidia_globs_len); + sc_mkdir_and_mount_and_glob_files(rootfs_dir, NATIVE_LIBDIR, + SC_LIBGL_DIR, nvidia_globs, + nvidia_globs_len); + sc_glob_files_only(rootfs_dir, NATIVE_LIBDIR "/nvidia*", SC_LIBGL_DIR, + nvidia_globs, nvidia_globs_len); +#if UINTPTR_MAX == 0xffffffffffffffff // Alternative 32-bit support - sc_mkdir_and_mount_and_glob_files(rootfs_dir, "/usr/lib32", + sc_mkdir_and_mount_and_glob_files(rootfs_dir, LIB32_DIR, SC_LIBGL32_DIR, nvidia_globs, nvidia_globs_len); - sc_glob_files_only(rootfs_dir, "/usr/lib32/nvidia*", + sc_glob_files_only(rootfs_dir, LIB32_DIR "/nvidia*", SC_LIBGL32_DIR, nvidia_globs, nvidia_globs_len); +#endif } #endif // ifdef NVIDIA_BIARCH From 7f8bc25afd4bf830e6dcbc24cd8334253f519c33 Mon Sep 17 00:00:00 2001 From: Neal Gompa Date: Tue, 30 Jan 2018 12:37:06 -0500 Subject: [PATCH 4/4] packaging/fedora: Enable support for the NVIDIA proprietary drivers Signed-off-by: Neal Gompa --- packaging/fedora/snapd.spec | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packaging/fedora/snapd.spec b/packaging/fedora/snapd.spec index 3f6376fe93e..8eca214918f 100644 --- a/packaging/fedora/snapd.spec +++ b/packaging/fedora/snapd.spec @@ -19,6 +19,11 @@ # For the moment, we don't support all golang arches... %global with_goarches 0 +# Set if multilib is enabled for supported arches +%ifarch x86_64 aarch64 %{power64} s390x +%global with_multilib 1 +%endif + %if ! %{with vendorized} %global with_bundled 0 %else @@ -430,6 +435,8 @@ autoreconf --force --install --verbose %configure \ --disable-apparmor \ --libexecdir=%{_libexecdir}/snapd/ \ + --enable-nvidia-biarch \ + %{?with_multilib:--with-32bit-libdir=%{_prefix}/lib} \ --with-snap-mount-dir=%{_sharedstatedir}/snapd/snap \ --with-merged-usr