diff --git a/orte/mca/schizo/flux/Makefile.am b/orte/mca/schizo/flux/Makefile.am new file mode 100644 index 00000000000..2a3100189f2 --- /dev/null +++ b/orte/mca/schizo/flux/Makefile.am @@ -0,0 +1,34 @@ +# +# Copyright (c) 2016 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +sources = \ + schizo_flux_component.c \ + schizo_flux.h \ + schizo_flux.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_orte_schizo_flux_DSO +component_noinst = +component_install = mca_schizo_flux.la +else +component_noinst = libmca_schizo_flux.la +component_install = +endif + +mcacomponentdir = $(ortelibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_schizo_flux_la_SOURCES = $(sources) +mca_schizo_flux_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_schizo_flux_la_SOURCES = $(sources) +libmca_schizo_flux_la_LDFLAGS = -module -avoid-version diff --git a/orte/mca/schizo/flux/owner.txt b/orte/mca/schizo/flux/owner.txt new file mode 100644 index 00000000000..85b4416d206 --- /dev/null +++ b/orte/mca/schizo/flux/owner.txt @@ -0,0 +1,7 @@ +# +# owner/status file +# owner: institution that is responsible for this package +# status: e.g. active, maintenance, unmaintained +# +owner: INTEL +status: active diff --git a/orte/mca/schizo/flux/schizo_flux.c b/orte/mca/schizo/flux/schizo_flux.c new file mode 100644 index 00000000000..7c6985674c2 --- /dev/null +++ b/orte/mca/schizo/flux/schizo_flux.c @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include "orte_config.h" +#include "orte/types.h" +#include "opal/types.h" + +#ifdef HAVE_UNISTD_H +#include +#endif +#include + +#include "opal/util/argv.h" +#include "opal/util/basename.h" +#include "opal/util/opal_environ.h" + +#include "orte/runtime/orte_globals.h" +#include "orte/util/name_fns.h" +#include "orte/mca/schizo/base/base.h" + +#include "schizo_flux.h" + +static orte_schizo_launch_environ_t check_launch_environment(void); +static void finalize(void); + +orte_schizo_base_module_t orte_schizo_flux_module = { + .check_launch_environment = check_launch_environment, + .finalize = finalize +}; + +static char **pushed_envs = NULL; +static char **pushed_vals = NULL; +static orte_schizo_launch_environ_t myenv; +static bool myenvdefined = false; + +static orte_schizo_launch_environ_t check_launch_environment(void) +{ + char *bind, *list, *ptr; + int i; + + if (myenvdefined) { + return myenv; + } + myenvdefined = true; + + /* we were only selected because FLUX was detected + * and we are an app, so no need to further check + * that here. Instead, see if we were direct launched + * vs launched via mpirun */ + if (NULL != orte_process_info.my_daemon_uri) { + /* nope */ + myenv = ORTE_SCHIZO_NATIVE_LAUNCHED; + opal_argv_append_nosize(&pushed_envs, OPAL_MCA_PREFIX"ess"); + opal_argv_append_nosize(&pushed_vals, "pmi"); + goto setup; + } + + /* see if we are in a FLUX allocation */ + if (NULL == getenv("FLUX_NODELIST")) { + /* nope */ + myenv = ORTE_SCHIZO_UNDETERMINED; + return myenv; + } + + /* we are in an allocation, but were we direct launched + * or are we a singleton? */ + if (NULL == getenv("FLUX_STEP_ID")) { + /* not in a job step - ensure we select the + * correct things */ + opal_argv_append_nosize(&pushed_envs, OPAL_MCA_PREFIX"ess"); + opal_argv_append_nosize(&pushed_vals, "singleton"); + myenv = ORTE_SCHIZO_MANAGED_SINGLETON; + goto setup; + } + myenv = ORTE_SCHIZO_DIRECT_LAUNCHED; + opal_argv_append_nosize(&pushed_envs, OPAL_MCA_PREFIX"ess"); + opal_argv_append_nosize(&pushed_vals, "pmi"); + + /* if we are direct launched by FLUX, then we want + * to ensure that we do not override their binding + * options, so set that envar */ + if (NULL != (bind = getenv("FLUX_CPU_BIND_TYPE"))) { + if (0 == strcmp(bind, "none")) { + opal_argv_append_nosize(&pushed_envs, OPAL_MCA_PREFIX"hwloc_base_binding_policy"); + opal_argv_append_nosize(&pushed_vals, "none"); + /* indicate we are externally bound so we won't try to do it ourselves */ + opal_argv_append_nosize(&pushed_envs, OPAL_MCA_PREFIX"orte_externally_bound"); + opal_argv_append_nosize(&pushed_vals, "1"); + } else if (bind == strstr(bind, "mask_cpu")) { + /* if the bind list is all F's, then the + * user didn't specify anything */ + if (NULL != (list = getenv("FLUX_CPU_BIND_LIST")) && + NULL != (ptr = strchr(list, 'x'))) { + ++ptr; // step over the 'x' + for (i=0; '\0' != *ptr; ptr++) { + if ('F' != *ptr) { + /* indicate we are externally bound */ + opal_argv_append_nosize(&pushed_envs, OPAL_MCA_PREFIX"orte_externally_bound"); + opal_argv_append_nosize(&pushed_vals, "1"); + break; + } + } + } + } + } + + setup: + opal_output_verbose(1, orte_schizo_base_framework.framework_output, + "schizo:flux DECLARED AS %s", orte_schizo_base_print_env(myenv)); + if (NULL != pushed_envs) { + for (i=0; NULL != pushed_envs[i]; i++) { + opal_setenv(pushed_envs[i], pushed_vals[i], true, &environ); + } + } + return myenv; +} + +static void finalize(void) +{ + int i; + + if (NULL != pushed_envs) { + for (i=0; NULL != pushed_envs[i]; i++) { + opal_unsetenv(pushed_envs[i], &environ); + } + opal_argv_free(pushed_envs); + opal_argv_free(pushed_vals); + } +} diff --git a/orte/mca/schizo/flux/schizo_flux.h b/orte/mca/schizo/flux/schizo_flux.h new file mode 100644 index 00000000000..5a648360a37 --- /dev/null +++ b/orte/mca/schizo/flux/schizo_flux.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef _MCA_SCHIZO_FLUX_H_ +#define _MCA_SCHIZO_FLUX_H_ + +#include "orte_config.h" + +#include "orte/types.h" + +#include "opal/mca/base/base.h" +#include "orte/mca/schizo/schizo.h" + + +BEGIN_C_DECLS + +ORTE_MODULE_DECLSPEC extern orte_schizo_base_component_t mca_schizo_flux_component; +extern orte_schizo_base_module_t orte_schizo_flux_module; + +END_C_DECLS + +#endif /* MCA_SCHIZO_FLUX_H_ */ diff --git a/orte/mca/schizo/flux/schizo_flux_component.c b/orte/mca/schizo/flux/schizo_flux_component.c new file mode 100644 index 00000000000..3084e78139d --- /dev/null +++ b/orte/mca/schizo/flux/schizo_flux_component.c @@ -0,0 +1,51 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2016 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "orte_config.h" +#include "orte/types.h" +#include "opal/types.h" + +#include "opal/util/show_help.h" + +#include "orte/mca/schizo/schizo.h" +#include "schizo_flux.h" + +static int component_query(mca_base_module_t **module, int *priority); + +/* + * Struct of function pointers and all that to let us be initialized + */ +orte_schizo_base_component_t mca_schizo_flux_component = { + .base_version = { + MCA_SCHIZO_BASE_VERSION_1_0_0, + .mca_component_name = "flux", + MCA_BASE_MAKE_VERSION(component, ORTE_MAJOR_VERSION, ORTE_MINOR_VERSION, + ORTE_RELEASE_VERSION), + .mca_query_component = component_query, + }, + .base_data = { + /* The component is checkpoint ready */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, +}; + +static int component_query(mca_base_module_t **module, int *priority) +{ + /* disqualify ourselves if we are not an app or under flux */ + if (!ORTE_PROC_IS_APP || NULL == getenv("FLUX_JOB_ID")) { + *priority = 0; + *module = NULL; + return OPAL_ERROR; + } + + *module = (mca_base_module_t*)&orte_schizo_flux_module; + *priority = 60; + return ORTE_SUCCESS; +}