Try using Preferences.jl instead of Pkg.build

.github/workflows/UnitTests.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, windows-latest, ubuntu-latest]
-        julia_version: ["1.3", "1.4", "1.5", "nightly"]
+        julia_version: ["1.6", "1", "nightly"]
         julia_arch: [x64, x86]
         exclude:
           - os: macOS-latest
@@ -62,8 +62,7 @@ jobs:
     strategy:
       matrix:
         mpi: [mpich, openmpi]
-        mpi_abi: ["", "unknown"]
-        julia_version: ["1.5", "nightly"]
+        julia_version: ["1", "nightly"]
 
       fail-fast: false
 
@@ -72,7 +71,6 @@ jobs:
     env:
       JULIA_MPI_BINARY: system
       JULIA_MPI_PATH: /usr/local
-      JULIA_MPI_ABI: ${{ matrix.mpi_abi }}
 
     steps:
     - name: Cancel Previous Runs
@@ -106,6 +104,9 @@ jobs:
           ${{ runner.os }}-
 
     - uses: julia-actions/julia-buildpkg@latest
+    - name: "Set binary" 
+      run: |
+        julia --project -e 'using MPI; MPI.use_system_binary()'
     - uses: julia-actions/julia-runtest@latest
 
 
@@ -114,16 +115,14 @@ jobs:
     strategy:
       matrix:
         mpi: [mpich, libopenmpi-dev]
-        mpi_abi: ["", "unknown"]
-        julia_version: ["1.5", "nightly"]
+        julia_version: ["1", "nightly"]
 
       fail-fast: false
 
     runs-on: ubuntu-20.04
 
     env:
       JULIA_MPI_BINARY: system
-      JULIA_MPI_ABI: ${{ matrix.mpi_abi }}
       OMPI_MCA_btl_base_warn_component_unused: 0
 
     steps:
@@ -158,14 +157,18 @@ jobs:
           ${{ runner.os }}-
 
     - uses: julia-actions/julia-buildpkg@latest
+    - name: "Set binary" 
+      run: |
+        julia --project -e 'using MPI; MPI.use_system_binary()'
+
     - uses: julia-actions/julia-runtest@latest
 
 
   test-intel-linux:
     timeout-minutes: 20
     strategy:
       matrix:
-        julia_version: ["1.5", "nightly"]
+        julia_version: ["1", "nightly"]
 
       fail-fast: false
 
@@ -243,7 +246,7 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
-        julia_version: ["1.5", "nightly"]
+        julia_version: ["1", "nightly"]
 
       fail-fast: false
 

Project.toml

@@ -11,6 +11,7 @@ MPICH_jll = "7cb0a576-ebde-5e09-9194-50597f1243b4"
 MicrosoftMPI_jll = "9237b28f-5490-5468-be7b-bb81f5f5e6cf"
 OpenMPI_jll = "fe0851c0-eecd-5654-98d4-656369965a5c"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Preferences = "21216c6a-2e73-6563-6e65-726566657250"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Requires = "ae029012-a4dd-5104-9daa-d747884805df"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
@@ -19,7 +20,7 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 [compat]
 DocStringExtensions = "0.8"
 Requires = "~0.5, 1.0"
-julia = "1"
+julia = "1.6"
 
 [extras]
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"

deps/build.jl

@@ -1,3 +1,6 @@
+# TODO: figure out if we can set preferences at Pkg.build time?
+exit()
+
 using Pkg.TOML, Libdl
 
 config_toml = joinpath(first(DEPOT_PATH), "prefs", "MPI.toml")

src/MPI.jl

@@ -1,7 +1,7 @@
 module MPI
 
 using Libdl, Serialization
-using Requires
+using Requires, Preferences
 using DocStringExtensions
 
 export mpiexec, UBuffer, VBuffer
@@ -32,12 +32,69 @@ function _doc_external(fname)
 """
 end
 
-try
-    include(joinpath(dirname(@__DIR__), "deps","deps.jl"))
-catch e
-    error("MPI.jl not properly configured, please run `Pkg.build(\"MPI\")`.")
+# preferences
+# binary = "system" | "OpenMPI_jll" | "MPICH_jll" | "MicrosoftMPI_jll"
+# if binary == "system" then the following also need to be set:
+#   libmpi = library name (optional path)
+#   abi = "openmpi" | "mpich" | "microsoftmpi" | "unknown"
+#   mpiexec = MPI launcher command
+
+
+const binary = @load_preference("binary", Sys.iswindows() ? "MicrosoftMPI_jll" : "MPICH_jll")
+
+@static if binary == "MPICH_jll"
+    using MPICH_jll
+    const abi = "MPICH"
+    const _mpiexec = MPICH_jll.mpiexec
+    const mpiexec_path = MPICH_jll.mpiexec_path
+    __init__deps() = nothing
+elseif binary == "OpenMPI_jll"
+    using OpenMPI_jll
+    const abi = "OpenMPI"
+    const _mpiexec = OpenMPI_jll.mpiexec
+    const mpiexec_path = OpenMPI_jll.mpiexec_path
+    __init__deps() = nothing
+elseif binary == "MicrosoftMPI_jll"
+    using MicrosoftMPI_jll
+    const abi = "MicrosoftMPI"
+    const _mpiexec = MicrosoftMPI_jll.mpiexec
+    const mpiexec_path = MicrosoftMPI_jll.mpiexec_path
+    __init__deps() = nothing
+elseif binary == "system"
+    const libmpi = @load_preference("libmpi")
+    const abi = @load_preference("abi")
+    const mpiexec_path = @load_preference("mpiexec")
+    const mpiexec_cmd = `$mpiexec_path`
+    _mpiexec(fn) = begin
+        fn(mpiexec_cmd)
+    end
+
+    function __init__deps()
+        libabi = identify_abi()
+        if libabi != abi
+            @set_preferences!("abi" => libabi)
+            error("MPI library ABI has changed; restart Julia for the change to take effect")
+        end
+    end        
+else
+    error("Invalid binary preference $binary")
+end
+
+if abi == "MPICH"
+    include("consts/mpich.jl")
+elseif abi == "OpenMPI"
+    include("consts/openmpi.jl")
+elseif abi == "MicrosoftMPI"
+    include("consts/microsoftmpi.jl")
+else
+    include(joinpath(dirname(@__DIR__), "deps", "consts.jl"))
 end
+
+
+
 include("implementations.jl")
+include("preferences.jl")
+include("identify.jl")
 include("error.jl")
 include("handle.jl")
 include("info.jl")

src/environment.jl

@@ -287,12 +287,10 @@ or `false`.
 function has_cuda()
     flag = get(ENV, "JULIA_MPI_HAS_CUDA", nothing)
     if flag === nothing
-        # Only Open MPI provides a function to check CUDA support
-        @static if MPI_LIBRARY == OpenMPI
-            # int MPIX_Query_cuda_support(void)
+        # Not every MPI implementation provides a function to check CUDA support
+        # It is broken on MPICH_jll: https://github.com/JuliaPackaging/Yggdrasil/issues/4039
+        if binary != "MPICH_jll" && dlsym_e(dlopen(libmpi), :MPIX_Query_cuda_support) != C_NULL
             return 0 != ccall((:MPIX_Query_cuda_support, libmpi), Cint, ())
-        elseif MPI_LIBRARY == IBMSpectrumMPI
-            return true
         else
             return false
         end

src/identify.jl

@@ -0,0 +1,87 @@
+"""
+    impl, version = identify_implementation(version_string)
+
+Attempt to identify the MPI implementation based on
+[`MPI_LIBRARY_VERSION_STRING`](@ref). Returns a tuple of values:
+
+- `impl`: a value of type [`MPIImpl`](@ref)
+- `version`: a `VersionNumber` of the library, or `nothing` if it cannot be determined.
+
+This function is only intended for internal use. Users should use [`MPI_LIBRARY`](@ref),
+[`MPI_LIBRARY_VERSION`](@ref).
+"""
+function identify_implementation(version_string::String)
+    impl = "UnknownMPI"
+    version = v"0"
+
+    if startswith(version_string, "MPICH")
+        impl = "MPICH"
+        # "MPICH Version:\t%s\n" /  "MPICH2 Version:\t%s\n"
+        if (m = match(r"^MPICH2? Version:\t(\d+.\d+.\d+\w*)\n", version_string)) !== nothing
+            version = VersionNumber(m.captures[1])
+        end
+
+    elseif startswith(version_string, "Open MPI")
+        # Open MPI / Spectrum MPI
+        impl = occursin("IBM Spectrum MPI", version_string) ? "IBMSpectrumMPI" : "OpenMPI"
+
+        if (m = match(r"^Open MPI v(\d+.\d+.\d+\w*)", version_string)) !== nothing
+            version = VersionNumber(m.captures[1])
+        end
+
+    elseif startswith(version_string, "Microsoft MPI")
+        impl = "MicrosoftMPI"
+        # "Microsoft MPI %u.%u.%u.%u%S"
+        # ignore last 2 (build numbers)
+        if (m = match(r"^Microsoft MPI (\d+.\d+)", version_string)) !== nothing
+            version = VersionNumber(m.captures[1])
+        end
+
+    elseif startswith(version_string, "Intel")
+        impl = "IntelMPI"
+
+        # "Intel(R) MPI Library 2019 Update 4 for Linux* OS"
+        if (m = match(r"^Intel\(R\) MPI Library (\d+)(?: Update (\d+))?", version_string)) !== nothing
+            if m.captures[2] === nothing
+                version = VersionNumber(m.captures[1])
+            else
+                version = VersionNumber(m.captures[1]*"."*m.captures[2])
+            end
+        end
+
+    elseif startswith(version_string, "MVAPICH2")
+        impl = "MVAPICH"
+        # "MVAPICH2 Version      :\t%s\n")
+        if (m = match(r"^MVAPICH2? Version\s*:\t(\S*)\n", version_string)) !== nothing
+            version = VersionNumber(m.captures[1])
+        end
+
+    elseif occursin("CRAY MPICH", version_string)
+        impl = "CrayMPICH"
+        # "MPI VERSION    : CRAY MPICH version 7.7.10 (ANL base 3.2)\n"
+        if (m = match(r"CRAY MPICH version (\d+.\d+.\d+)", version_string)) !== nothing
+            version = VersionNumber(m.captures[1])
+        end
+    end
+
+    return impl, version
+end
+identify_implementation() = identify_implementation(Get_library_version())
+
+function identify_abi(impl::String, version::VersionNumber)
+    if (impl == "MPICH" && version >= v"3.1" ||
+        impl == "IntelMPI" && version > v"2014" ||
+        impl == "MVAPICH" && version >= v"2" ||
+        impl == "CrayMPICH" && version >= v"7")
+        # https://www.mpich.org/abi/
+        abi = "MPICH"
+    elseif impl == "OpenMPI" || impl == "IBMSpectrumMPI"
+        abi = "OpenMPI"
+    elseif impl == "MicrosoftMPI"
+        abi = "MicrosoftMPI"
+    else
+        abi = "unknown"
+    end
+end
+
+identify_abi() = identify_abi(identify_implementation()...)