diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py
index a91fbb7aecf..c3ff5534e87 100644
--- a/python/libcudf/libcudf/load.py
+++ b/python/libcudf/libcudf/load.py
@@ -76,9 +76,15 @@ def load_library():
         # Prefer the libraries bundled in this package. If they aren't found
         # (which might be the case in builds where the library was prebuilt before
         # packaging the wheel), look for a system installation.
-        libcudf_lib = _load_wheel_installation(soname)
-        if libcudf_lib is None:
-            libcudf_lib = _load_system_installation(soname)
+        try:
+            libcudf_lib = _load_wheel_installation(soname)
+            if libcudf_lib is None:
+                libcudf_lib = _load_system_installation(soname)
+        except OSError:
+            # If none of the searches above succeed, just silently return None
+            # and rely on other mechanisms (like RPATHs on other DSOs) to
+            # help the loader find the library.
+            pass
 
     # The caller almost never needs to do anything with this library, but no
     # harm in offering the option since this object at least provides a handle