wreck: add wreck plugin for spectrum MPI

Add mpi "personality" for IBM spectrum MPI, enabled by user with
wreckrun -o mpi-spectrum.  Note that this plugin assumes MPI is
installed in /opt/ibm/spectrum_mpi.  It also disables PAMI, the
spectrum enhanced collectives due to their dependency on the RM
providing a PMIx server. See flux-framework#1382 for further details.  It also sets
the soft stack limit to a value the MPI runtime seems to require.

See flux-framework#1382 for more details.

Fixes flux-framework#1584

src/modules/wreck/Makefile.am

@@ -81,7 +81,8 @@ dist_wreckscripts_SCRIPTS = \
 	lua.d/mvapich.lua \
 	lua.d/pmi-mapping.lua \
 	lua.d/intel_mpi.lua \
-	lua.d/openmpi.lua
+        lua.d/openmpi.lua \
+        lua.d/spectrum.lua
 
 # XXX: Hack below to force rebuild of unbuilt wrexecd dependencies
 #

src/modules/wreck/lua.d/spectrum.lua

@@ -0,0 +1,49 @@
+-- Set environment specific to spectrum_mpi (derived from openmpi)
+--
+
+if wreck:getopt ("mpi") ~= "spectrum" then return end
+
+local posix = require 'posix'
+
+function prepend_path (env_var, path)
+    local env = wreck.environ
+    if env[env_var] == nil then
+       suffix = ''
+    else
+       suffix = ':'..env[env_var]
+    end
+    env[env_var] = path..suffix
+end
+
+function rexecd_init ()
+    local env = wreck.environ
+    local f = wreck.flux
+    local rundir = f:getattr ('broker.rundir')
+
+    -- Avoid shared memory segment name collisions
+    -- when flux instance runs >1 broker per node.
+    env['OMPI_MCA_orte_tmpdir_base'] = rundir
+
+    -- Assumes the installation paths of Spectrum MPI on LLNL's Sierra
+    env['OMPI_MCA_osc'] = "pt2pt"
+    env['OMPI_MCA_pml'] = "yalla"
+    env['OMPI_MCA_btl'] = "self"
+    env['MPI_ROOT'] = "/opt/ibm/spectrum_mpi"
+    env['OPAL_LIBDIR'] = "/opt/ibm/spectrum_mpi/lib"
+    env['OMPI_MCA_coll_hcoll_enable'] = '0'
+
+    env['PMIX_SERVER_URI'] = nil
+    env['PMIX_SERVER_URI2'] = nil
+
+    -- Help find libcollectives.so
+    prepend_path ('LD_LIBRARY_PATH', '/opt/ibm/spectrum_mpi/lib/pami_port')
+    prepend_path ('LD_PRELOAD', '/opt/ibm/spectrum_mpi/lib/libpami_cudahook.so')
+end
+
+function rexecd_task_init ()
+    -- Approximately `ulimit -Ss 10240`
+    -- Used to silence IBM MCM warnings
+    posix.setrlimit ("stack", 10485760)
+end
+
+-- vi: ts=4 sw=4 expandtab