diff --git a/src/bindings/lua/wreck.lua b/src/bindings/lua/wreck.lua index 1d286c3fb0ed..98b960f2697b 100644 --- a/src/bindings/lua/wreck.lua +++ b/src/bindings/lua/wreck.lua @@ -97,7 +97,7 @@ local function get_filtered_env () for k,v in pairs (env) do if k:match ("SLURM_") then env[k] = nil end if k:match ("FLUX_API") then env[k] = nil end - if k:match ("FLUX_TMPDIR") then env[k] = nil end + if k:match ("FLUX_URI") then env[k] = nil end end -- XXX: MVAPICH2 at least requires MPIRUN_RSH_LAUNCH to be set -- in the environment or PMI doesn't work (for unknown reason) diff --git a/src/bindings/python/flux/command_helpers.py b/src/bindings/python/flux/command_helpers.py index e6998ab1f0af..dd4c2f1a0825 100644 --- a/src/bindings/python/flux/command_helpers.py +++ b/src/bindings/python/flux/command_helpers.py @@ -1,22 +1,19 @@ from __future__ import print_function import sys, os, re, flux -def list_instances(list_all, top_only, sid=None): +def list_instances(sid=None): tmpdir = '/tmp' tmpdir = os.environ.get('TMPDIR', tmpdir) - if not list_all: - tmpdir = os.environ.get('FLUX_TMPDIR', tmpdir) - fdir = re.compile('flux-(?P[^-]+)-0') + fdir = re.compile('flux-(?P[^-]+)-') - # Sometimes flux tmpdirs end up in sub-directories of previous tmpdirs for dirname, dirs, files in os.walk(tmpdir, topdown=True): for m in [fdir.match(d) for d in dirs]: if not m: continue - if sid is not None and not re.search('flux-' + sid + '-0', + if sid is not None and not re.search('flux-' + sid + '-', m.string): continue - job = os.path.join(tmpdir, m.string) + job = os.path.join (os.path.join(tmpdir, m.string), '0') uri = 'local://' + job try: with open(os.path.join(job, 'broker.pid')) as f: @@ -26,5 +23,4 @@ def list_instances(list_all, top_only, sid=None): yield (m.group('id'), uri) except: pass - if top_only: - break + break diff --git a/src/bindings/python/test_commands/sideflux.py b/src/bindings/python/test_commands/sideflux.py index 364635205b8c..d5d2087f2442 100644 --- a/src/bindings/python/test_commands/sideflux.py +++ b/src/bindings/python/test_commands/sideflux.py @@ -45,18 +45,15 @@ def __init__(self, size=1): global flux_exe self.size = size self.tmpdir = tempfile.mkdtemp(prefix='flux-sandbox-') - self.flux_tmpdir = os.path.join(self.tmpdir, 'flux-sideflux-0') - self.flux_uri = 'local://' + self.flux_tmpdir - os.makedirs(self.flux_tmpdir) + self.flux_uri = 'local://' + self.tmpdir + '/0' self.cleaned = False def start(self): flux_command = [flux_exe, 'start', '--size={}'.format(self.size), '-o', - '-L,stderr,--sid,sideflux', 'bash'] + '-L,stderr,--socket-directory,' + self.tmpdir , 'bash'] # print ' '.join(flux_command) FNULL = open(os.devnull, 'w+') self.subenv = os.environ.copy() - self.subenv.pop('FLUX_TMPDIR', None) self.subenv.pop('FLUX_URI', None) self.subenv['TMPDIR'] = self.tmpdir self.sub = subprocess.Popen( @@ -72,7 +69,6 @@ def start(self): print('echo READY', file=self.sub.stdin) self.env_items = {} - # self.env_items['FLUX_TMPDIR'] = self.flux_tmpdir self.env_items['FLUX_URI'] = self.flux_uri while True: diff --git a/src/broker/broker.c b/src/broker/broker.c index 90443084b41c..941c7ef650c1 100644 --- a/src/broker/broker.c +++ b/src/broker/broker.c @@ -93,6 +93,10 @@ typedef struct { uint32_t size; /* session size */ uint32_t rank; /* our rank in session */ char *sid; /* session id */ + char *socket_dir; + char *local_uri; + char *parent_uri; + /* Modules */ modhash_t *modhash; @@ -152,8 +156,8 @@ static void load_modules (ctx_t *ctx, zlist_t *modules, zlist_t *modopts, zhash_t *modexclude, const char *modpath); static void update_proctitle (ctx_t *ctx); -static void update_environment (ctx_t *ctx); -static void update_pidfile (ctx_t *ctx, bool force); +static void create_rankdir (ctx_t *ctx); +static void update_pidfile (ctx_t *ctx); static void rank0_shell (ctx_t *ctx); static int rank0_shell_exit_handler (struct subprocess *p, void *arg); @@ -162,7 +166,7 @@ static void boot_local (ctx_t *ctx); static const struct flux_handle_ops broker_handle_ops; -#define OPTIONS "t:vqR:S:p:M:X:L:N:Pk:e:r:s:c:fnH:O:x:T:g:" +#define OPTIONS "t:vqR:S:p:M:X:L:N:Pk:e:r:s:c:nH:O:x:T:g:D:" static const struct option longopts[] = { {"sid", required_argument, 0, 'N'}, {"child-uri", required_argument, 0, 't'}, @@ -186,6 +190,7 @@ static const struct option longopts[] = { {"heartrate", required_argument, 0, 'H'}, {"timeout", required_argument, 0, 'T'}, {"shutdown-grace", required_argument, 0, 'g'}, + {"socket-directory",required_argument, 0, 'D'}, {0, 0, 0, 0}, }; @@ -212,10 +217,10 @@ static void usage (void) " -k,--k-ary K Wire up in a k-ary tree\n" " -c,--command string Run command on rank 0\n" " -n,--noshell Do not spawn a shell even if on a tty\n" -" -f,--force Kill rival broker and start\n" " -H,--heartrate SECS Set heartrate in seconds (rank 0 only)\n" " -T,--timeout SECS Set wireup timeout in seconds (rank 0 only)\n" " -g,--shutdown-grace SECS Set shutdown grace period in seconds\n" +" -D,--socket-directory DIR Create ipc sockets in DIR (local bootstrap)\n" ); exit (1); } @@ -224,7 +229,6 @@ int main (int argc, char *argv[]) { int c; ctx_t ctx; - bool fopt = false; bool nopt = false; zlist_t *modules, *modopts; zhash_t *modexclude; @@ -340,9 +344,6 @@ int main (int argc, char *argv[]) case 'n': /* --noshell */ nopt = true; break; - case 'f': /* --force */ - fopt = true; - break; case 'H': /* --heartrate SECS */ if (heartbeat_set_ratestr (ctx.heartbeat, optarg) < 0) err_exit ("heartrate `%s'", optarg); @@ -353,6 +354,17 @@ int main (int argc, char *argv[]) case 'g': /* --shutdown-grace SECS */ ctx.shutdown_grace = strtod (optarg, NULL); break; + case 'D': { /* --socket-directory DIR */ + struct stat sb; + if (stat (optarg, &sb) < 0) + err_exit ("%s", optarg); + if (!S_ISDIR (sb.st_mode)) + msg_exit ("%s: not a directory", optarg); + if ((sb.st_mode & S_IRWXU) != S_IRWXU) + msg_exit ("%s: invalid mode: 0%o", optarg, sb.st_mode); + ctx.socket_dir = xstrdup (optarg); + break; + } default: usage (); } @@ -380,7 +392,6 @@ int main (int argc, char *argv[]) /* Process config from the KVS of enclosing instance (if any) * and not forced to use a config file by the command line. - * (FLUX_TMPDIR has not yet been overridden within this instance) */ ctx.cf = flux_conf_create (); if (!(confdir = getenv ("FLUX_CONF_DIRECTORY"))) @@ -391,7 +402,7 @@ int main (int argc, char *argv[]) msg ("Loading config from %s", confdir); if (flux_conf_load (ctx.cf) < 0 && errno != ESRCH) err_exit ("%s", confdir); - } else if (getenv ("FLUX_TMPDIR")) { + } else if (getenv ("FLUX_URI")) { flux_t h; if (ctx.verbose) msg ("Loading config from KVS"); @@ -400,6 +411,11 @@ int main (int argc, char *argv[]) if (kvs_conf_load (h, ctx.cf) < 0) err_exit ("could not load config from KVS"); flux_close (h); + /* Stash FLUX_URI value for later use, but unset it in the environment + * so a connection to the enclosing instance is not made inadvertantly. + */ + ctx.parent_uri = xstrdup (getenv ("FLUX_URI")); + unsetenv ("FLUX_URI"); } /* Arrange to load config entries into kvs config.* @@ -480,6 +496,23 @@ int main (int argc, char *argv[]) } if (!ctx.sid) ctx.sid = xstrdup ("0"); + + /* Now that we know the rank (either from command line or PMI, + * create a subdirectory of socket_dir for the sockets and pidfile + * specific to this rank of the broker. + */ + if (!ctx.socket_dir) { + char *tmpdir = getenv ("TMPDIR"); + char *template = xasprintf ("%s/flux-%s-XXXXXX", + tmpdir ? tmpdir : "/tmp", ctx.sid); + + if (!(ctx.socket_dir = mkdtemp (template))) + err_exit ("mkdtemp %s", template); + cleanup_push_string (cleanup_directory, ctx.socket_dir); + } + + create_rankdir (&ctx); + /* If we're missing the wiring, presume that the session is to be * started on a single node and compute appropriate ipc:/// sockets. */ @@ -523,8 +556,7 @@ int main (int argc, char *argv[]) } update_proctitle (&ctx); - update_environment (&ctx); - update_pidfile (&ctx, fopt); + update_pidfile (&ctx); if (!nopt && ctx.rank == 0 && (isatty (STDIN_FILENO) || ctx.shell_cmd)) { ctx.shell = subprocess_create (ctx.sm); @@ -622,6 +654,11 @@ int main (int argc, char *argv[]) zlist_destroy (&modopts); /* autofree set */ zhash_destroy (&modexclude); /* values const (no destructor) */ + if (ctx.parent_uri) + free (ctx.parent_uri); + if (ctx.local_uri) + free (ctx.local_uri); + return 0; } @@ -678,51 +715,32 @@ static void update_proctitle (ctx_t *ctx) ctx->proctitle = s; } -static void update_environment (ctx_t *ctx) +/* The 'ranktmp' dir will contain the broker.pid file and local:// socket. + * It will be created in ctx->socket_dir. + */ +static void create_rankdir (ctx_t *ctx) { - const char *oldtmp = flux_get_tmpdir (); - static char tmpdir[PATH_MAX + 1]; - - (void)snprintf (tmpdir, sizeof (tmpdir), "%s/flux-%s-%d", - oldtmp, ctx->sid, ctx->rank); - if (mkdir (tmpdir, 0700) < 0 && errno != EEXIST) - err_exit ("mkdir %s", tmpdir); - if (ctx->verbose) - msg ("FLUX_TMPDIR: %s", tmpdir); - if (flux_set_tmpdir (tmpdir) < 0) - err_exit ("flux_set_tmpdir"); - cleanup_push_string(cleanup_directory, tmpdir); + char *ranktmp = xasprintf ("%s/%d", ctx->socket_dir, ctx->rank); + + if (mkdir (ranktmp, 0700) < 0) + err_exit ("mkdir %s", ranktmp); + cleanup_push_string (cleanup_directory, ranktmp); + ctx->local_uri = xasprintf ("local://%s", ranktmp); + free (ranktmp); } -static void update_pidfile (ctx_t *ctx, bool force) +static void update_pidfile (ctx_t *ctx) { - const char *tmpdir = flux_get_tmpdir (); - char *pidfile; - pid_t pid; + char *pidfile = xasprintf ("%s/%d/broker.pid", ctx->socket_dir, ctx->rank); FILE *f; - if (asprintf (&pidfile, "%s/broker.pid", tmpdir) < 0) - oom (); - if ((f = fopen (pidfile, "r"))) { - if (fscanf (f, "%u", &pid) == 1 && kill (pid, 0) == 0) { - if (force) { - if (kill (pid, SIGKILL) < 0) - err_exit ("kill %d", pid); - msg ("killed broker with pid %d", pid); - } else - msg_exit ("broker is already running in %s, pid %d", tmpdir, pid); - } - (void)fclose (f); - } if (!(f = fopen (pidfile, "w+"))) err_exit ("%s", pidfile); if (fprintf (f, "%u", getpid ()) < 0) err_exit ("%s", pidfile); if (fclose(f) < 0) err_exit ("%s", pidfile); - if (ctx->verbose) - msg ("pidfile: %s", pidfile); - cleanup_push_string(cleanup_file, pidfile); + cleanup_push_string (cleanup_file, pidfile); free (pidfile); } @@ -757,6 +775,7 @@ static void rank0_shell (ctx_t *ctx) subprocess_argv_append (ctx->shell, ctx->shell_cmd); } subprocess_set_environ (ctx->shell, environ); + subprocess_setenv (ctx->shell, "FLUX_URI", ctx->local_uri, 1); if (!ctx->quiet) flux_log (ctx->h, LOG_INFO, "starting shell"); @@ -821,25 +840,24 @@ static void boot_pmi (ctx_t *ctx) static void boot_local (ctx_t *ctx) { - const char *tmpdir = flux_get_tmpdir (); int rrank = ctx->rank == 0 ? ctx->size - 1 : ctx->rank - 1; - char * reqfile = xasprintf("%s/flux-%s-%d-req", tmpdir, ctx->sid, ctx->rank); - char * eventfile = xasprintf("%s/flux-%s-event", tmpdir, ctx->sid); + char *reqfile = xasprintf ("%s/%d/req", ctx->socket_dir, ctx->rank); + char *eventfile = xasprintf ("%s/event", ctx->socket_dir); overlay_set_child (ctx->overlay, "ipc://%s", reqfile); - cleanup_push_string(cleanup_file, reqfile); - free(reqfile); + cleanup_push_string (cleanup_file, reqfile); + free (reqfile); if (ctx->rank > 0) { int prank = ctx->k_ary == 0 ? 0 : (ctx->rank - 1) / ctx->k_ary; - overlay_push_parent (ctx->overlay, "ipc://%s/flux-%s-%d-req", - tmpdir, ctx->sid, prank); + overlay_push_parent (ctx->overlay, "ipc://%s/%d/req", + ctx->socket_dir, prank); } overlay_set_event (ctx->overlay, "ipc://%s", eventfile); if (ctx->rank == 0) { - cleanup_push_string(cleanup_file, eventfile); + cleanup_push_string (cleanup_file, eventfile); } - free(eventfile); - overlay_set_right (ctx->overlay, "ipc://%s/flux-%s-%d-req", - tmpdir, ctx->sid, rrank); + free (eventfile); + overlay_set_right (ctx->overlay, "ipc://%s/%d/req", + ctx->socket_dir, rrank); } static bool nodeset_suffix_member (char *name, uint32_t rank) @@ -1183,13 +1201,13 @@ static int cmb_exec_cb (zmsg_t **zmsg, void *arg) if (val != NULL) subprocess_setenv (p, iter.key, val, 1); } - /* - * Override key FLUX environment variables in env array - */ - subprocess_setenv (p, "FLUX_TMPDIR", getenv ("FLUX_TMPDIR"), 1); } else subprocess_set_environ (p, environ); + /* + * Override key FLUX environment variables in env array + */ + subprocess_setenv (p, "FLUX_URI", ctx->local_uri, 1); if (json_object_object_get_ex (request, "cwd", &o) && o != NULL) { const char *dir = json_object_get_string (o); @@ -1340,10 +1358,14 @@ static int cmb_getattr_cb (zmsg_t **zmsg, void *arg) } if (!strcmp (name, "snoop-uri")) val = snoop_get_uri (ctx->snoop); - else if (!strcmp (name, "parent-uri")) + else if (!strcmp (name, "tbon-parent-uri")) val = overlay_get_parent (ctx->overlay); - else if (!strcmp (name, "request-uri")) + else if (!strcmp (name, "tbon-request-uri")) val = overlay_get_child (ctx->overlay); + else if (!strcmp (name, "local-uri")) + val = ctx->local_uri; + else if (!strcmp (name, "parent-uri")) + val = ctx->parent_uri; else errno = ENOENT; if (!val) diff --git a/src/cmd/flux-comms.c b/src/cmd/flux-comms.c index 7784993a9159..f9a1f23695ca 100644 --- a/src/cmd/flux-comms.c +++ b/src/cmd/flux-comms.c @@ -48,7 +48,7 @@ void usage (void) "Usage: flux-comms [-r N] idle\n" " flux-comms [-r N] getattr attr\n" " flux-comms info\n" -" flux-comms [-r N] reparent new-parent-uri\n" +" flux-comms [-r N] reparent new-uri\n" " flux-comms [-r N] panic [msg ...]\n" " flux-comms [-r N] failover\n" " flux-comms [-r N] recover\n" @@ -104,7 +104,7 @@ int main (int argc, char *argv[]) } else if (!strcmp (cmd, "getattr")) { char *s; if (optind != argc - 1) - msg_exit ("getattr snoop-uri, parent-uri, or request-uri"); + msg_exit ("Usage: flux comms getattr attrname"); if (!(s = flux_getattr (h, rank, argv[optind]))) err_exit ("%s", argv[optind]); printf ("%s\n", s); diff --git a/src/cmd/flux-config.c b/src/cmd/flux-config.c index 232e55fa18a8..7205509d06a0 100644 --- a/src/cmd/flux-config.c +++ b/src/cmd/flux-config.c @@ -93,7 +93,7 @@ int main (int argc, char *argv[]) msg ("Loading config from %s", flux_conf_get_directory (cf)); if (flux_conf_load (cf) < 0) err_exit ("%s", flux_conf_get_directory (cf)); - } else if (getenv ("FLUX_TMPDIR")) { + } else if (getenv ("FLUX_URI")) { if (vopt) msg ("Loading config from KVS"); if (!(h = flux_open (NULL, 0))) diff --git a/src/cmd/flux-list-instances b/src/cmd/flux-list-instances index 96f8b8237eee..e4f1ab4807aa 100755 --- a/src/cmd/flux-list-instances +++ b/src/cmd/flux-list-instances @@ -6,16 +6,8 @@ from __future__ import print_function import sys, os, re, flux.command_helpers, argparse parser = argparse.ArgumentParser(description='List local flux instances') -parser.add_argument('--top_only', - action='store_true', - help='List only top-level instances, ignore children') -parser.add_argument( - '--all', '-a', - action='store_true', - help= - 'List all instances, including those outside of the enclosing instance') if __name__ == '__main__': args = parser.parse_args() - for sid, uri in flux.command_helpers.list_instances(args.all, args.top_only): + for sid, uri in flux.command_helpers.list_instances(): print("{}: {}".format(sid, uri)) diff --git a/src/cmd/flux-start.c b/src/cmd/flux-start.c index 14bf2baefd24..0756a46a4ef2 100644 --- a/src/cmd/flux-start.c +++ b/src/cmd/flux-start.c @@ -34,6 +34,7 @@ #include "src/common/libutil/xzmalloc.h" #include "src/common/libutil/log.h" #include "src/common/libutil/optparse.h" +#include "src/common/libutil/cleanup.h" int start_direct (optparse_t p, const char *cmd); void start_slurm (optparse_t p, const char *cmd); @@ -200,6 +201,18 @@ char *args_str (char *argz, size_t argz_len) return cpy; } +char *create_socket_dir (const char *sid) +{ + char *tmpdir = getenv ("TMPDIR"); + char *sockdir = xasprintf ("%s/flux-%s-XXXXXX", + tmpdir ? tmpdir : "/tmp", sid); + + if (!mkdtemp (sockdir)) + err_exit ("mkdtemp %s", sockdir); + cleanup_push_string (cleanup_directory, sockdir); + return sockdir; +} + int start_direct (optparse_t p, const char *cmd) { int size = optparse_get_int (p, "size", default_size); @@ -210,7 +223,8 @@ int start_direct (optparse_t p, const char *cmd) pid_t *pids; int reaped = 0; int rc = 0; - pid_t start_pid = getpid (); /* use as session id */ + char *sid = xasprintf ("%d", getpid ()); + char *sockdir = create_socket_dir (sid); if (!broker_path) msg_exit ("FLUX_BROKER_PATH is not set"); @@ -224,7 +238,8 @@ int start_direct (optparse_t p, const char *cmd) add_arg (&argz, &argz_len, "%s", broker_path); add_arg (&argz, &argz_len, "--size=%d", size); add_arg (&argz, &argz_len, "--rank=%d", rank); - add_arg (&argz, &argz_len, "--sid=%d", start_pid); + add_arg (&argz, &argz_len, "--sid=%s", sid); + add_arg (&argz, &argz_len, "--socket-directory=%s", sockdir); if (broker_opts) add_args_sep (&argz, &argz_len, broker_opts, ','); if (rank == 0 && cmd) @@ -284,7 +299,9 @@ int start_direct (optparse_t p, const char *cmd) } } + free (sid); free (pids); + free (sockdir); return (rc); } diff --git a/src/cmd/flux.c b/src/cmd/flux.c index 04fb1b36cca3..3374aae99032 100644 --- a/src/cmd/flux.c +++ b/src/cmd/flux.c @@ -51,9 +51,8 @@ void setup_path (flux_conf_t cf, const char *argv0); static void print_environment(flux_conf_t cf, const char * prefix); void setup_broker_env (flux_conf_t cf, const char *path_override); -#define OPTIONS "+T:tx:hM:O:B:vc:L:P:C:FS:u:" +#define OPTIONS "+tx:hM:O:B:vc:L:P:C:FS:u:" static const struct option longopts[] = { - {"tmpdir", required_argument, 0, 'T'}, {"trace-handle", no_argument, 0, 't'}, {"exec-path", required_argument, 0, 'x'}, {"module-path", required_argument, 0, 'M'}, @@ -81,11 +80,10 @@ static void usage (void) " -L,--lua-path PATH prepend PATH to LUA_PATH\n" " -P,--python-path PATH prepend PATH to PYTHONPATH\n" " -C,--lua-cpath PATH prepend PATH to LUA_CPATH\n" -" -T,--tmpdir PATH set FLUX_TMPDIR\n" " -t,--trace-handle set FLUX_HANDLE_TRACE=1 before executing COMMAND\n" " -B,--broker-path FILE override path to flux broker\n" " -c,--config DIR set path to config directory\n" -" -F,--file-config force use of config file, even if FLUX_TMPDIR set\n" +" -F,--file-config force use of config file, even if FLUX_URI is set\n" " -S,--secdir DIR set the directory where CURVE keys will be stored\n" " -u,--uri URI override default URI to flux broker\n" " -v,--verbose show FLUX_* environment and command search\n" @@ -145,11 +143,6 @@ int main (int argc, char *argv[]) case 'F': /* --file-config */ Fopt = true; break; - case 'T': /* --tmpdir PATH */ - if (setenv ("FLUX_TMPDIR", optarg, 1) < 0) - err_exit ("setenv"); - flux_conf_environment_set (cf, "FLUX_TMPDIR", optarg, ""); - break; case 't': /* --trace-handle */ if (setenv ("FLUX_HANDLE_TRACE", "1", 1) < 0) err_exit ("setenv"); @@ -209,7 +202,7 @@ int main (int argc, char *argv[]) * It is not an error if config is not foud in either place, we will * try to make do with compiled-in defaults. */ - if (!Fopt && getenv ("FLUX_TMPDIR") + if (!Fopt && getenv ("FLUX_URI") && !(argc > 0 && !strcmp (argv[0], "start"))) { flux_t h; flux_conf_load (cf); diff --git a/src/connectors/local/local.c b/src/connectors/local/local.c index 02e1b6357932..92bda7fb5908 100644 --- a/src/connectors/local/local.c +++ b/src/connectors/local/local.c @@ -186,8 +186,8 @@ static bool pidcheck (const char *pidfile) return running; } -/* Path is interpreted as the directory containing the unix domain socket. - * If NULL, flux_get_tmpdir() is used. +/* Path is interpreted as the directory containing the unix domain socket + * and broker pid. */ flux_t connector_init (const char *path, int flags) { @@ -197,10 +197,11 @@ flux_t connector_init (const char *path, int flags) char sockfile[PATH_MAX + 1]; int n, count; - if (!path) - path = flux_get_tmpdir (); - - n = snprintf (sockfile, sizeof (sockfile), "%s/flux-api", path); + if (!path) { + errno = EINVAL; + goto error; + } + n = snprintf (sockfile, sizeof (sockfile), "%s/local", path); if (n >= sizeof (sockfile)) { errno = EINVAL; goto error; diff --git a/src/modules/connector-local/local.c b/src/modules/connector-local/local.c index c7a360a7c37e..e4b5aae792ac 100644 --- a/src/modules/connector-local/local.c +++ b/src/modules/connector-local/local.c @@ -707,15 +707,21 @@ const int htablen = sizeof (htab) / sizeof (htab[0]); int mod_main (flux_t h, int argc, char **argv) { ctx_t *ctx = getctx (h); - char *sockpath = NULL, *dfltpath = NULL; + char sockpath[PATH_MAX + 1]; + char *local_uri = NULL; + char *tmpdir; int rc = -1; - /* Parse args. - */ - if (argc > 0) - sockpath = argv[0]; - if (!sockpath) - sockpath = dfltpath = xasprintf ("%s/flux-api", flux_get_tmpdir ()); + if (!(local_uri = flux_getattr (h, FLUX_NODEID_ANY, "local-uri"))) { + flux_log (h, LOG_ERR, "flux_getattr local-uri: %s", strerror (errno)); + goto done; + } + if (!(tmpdir = strstr (local_uri, "local://"))) { + flux_log (h, LOG_ERR, "malformed local-uri"); + goto done; + } + tmpdir += strlen ("local://"); + snprintf (sockpath, sizeof (sockpath), "%s/local", tmpdir); /* Create listen socket and watcher to handle new connections */ @@ -744,8 +750,8 @@ int mod_main (flux_t h, int argc, char **argv) } rc = 0; done: - if (dfltpath) - free (dfltpath); + if (local_uri) + free (local_uri); flux_msg_watcher_delvec (h, htab); flux_fd_watcher_destroy (ctx->listen_w); if (ctx->listen_fd >= 0) { diff --git a/src/modules/live/live.c b/src/modules/live/live.c index 9541be1538ce..42b58f0abdb2 100644 --- a/src/modules/live/live.c +++ b/src/modules/live/live.c @@ -275,7 +275,7 @@ static void parents_fromjson (ctx_t *ctx, JSON ar) if (i == 0) { if (p->uri) /* unlikely */ free (p->uri); - p->uri = flux_getattr (ctx->h, -1, "parent-uri"); + p->uri = flux_getattr (ctx->h, -1, "tbon-parent-uri"); } if (zlist_append (ctx->parents, p) < 0) oom (); diff --git a/src/modules/wreck/wrexec.c b/src/modules/wreck/wrexec.c index e617c4f189a0..8a0985a7fb25 100644 --- a/src/modules/wreck/wrexec.c +++ b/src/modules/wreck/wrexec.c @@ -43,6 +43,7 @@ struct rexec_ctx { int nodeid; flux_t h; char *wrexecd_path; + char *local_uri; }; struct rexec_session { @@ -57,6 +58,8 @@ struct rexec_session { static void freectx (void *arg) { struct rexec_ctx *ctx = arg; + if (ctx->local_uri) + free (ctx->local_uri); free (ctx); } @@ -81,6 +84,8 @@ static struct rexec_ctx *getctx (flux_t h) ctx = xzmalloc (sizeof (*ctx)); ctx->h = h; ctx->nodeid = flux_rank (h); + if (!(ctx->local_uri = flux_getattr (h, FLUX_NODEID_ANY, "local-uri"))) + err_exit ("flux_getattr local-uri"); flux_aux_set (h, "wrexec", ctx, freectx); kvs_watch_string (h, "config.wrexec.wrexecd_path", wrexec_path_set, (void *) ctx); @@ -163,6 +168,8 @@ static void exec_handler (struct rexec_ctx *ctx, uint64_t id, int *pfds) dup2 (pfds[0], 3); closeall (4); flux_log (ctx->h, LOG_DEBUG, "running %s %s %s", args[0], args[1], args[2]); + if (setenv ("FLUX_URI", ctx->local_uri, 1) < 0) + err_exit ("setenv"); if (execvp (args[0], args) < 0) { close (3); err_exit ("execvp"); diff --git a/src/modules/wreck/wrexecd.c b/src/modules/wreck/wrexecd.c index 05e315b4be78..c4457b949c6f 100644 --- a/src/modules/wreck/wrexecd.c +++ b/src/modules/wreck/wrexecd.c @@ -1063,7 +1063,7 @@ int exec_command (struct prog_ctx *ctx, int i) ctx->in_task = 1; lua_stack_call (ctx->lua_stack, "rexecd_task_init"); - prog_ctx_setenv (ctx, "FLUX_TMPDIR", getenv ("FLUX_TMPDIR")); + prog_ctx_setenv (ctx, "FLUX_URI", getenv ("FLUX_URI")); prog_ctx_setenvf (ctx, "MPIRUN_RANK", 1, "%d", t->globalid); prog_ctx_setenvf (ctx, "PMI_RANK", 1, "%d", t->globalid); prog_ctx_setenvf (ctx, "FLUX_TASK_RANK", 1, "%d", t->globalid); diff --git a/t/t0005-exec.t b/t/t0005-exec.t index b59a3898a995..525abc426ef4 100755 --- a/t/t0005-exec.t +++ b/t/t0005-exec.t @@ -44,9 +44,9 @@ test_expect_success 'flux exec passes environment variables' ' test_on_rank 0 sh -c "test \"\$FOOTEST\" = \"t\"" ' -test_expect_success 'flux exec does not pass FLUX_TMPDIR' ' - # Ensure FLUX_TMPDIR for rank 1 doesn not equal FLUX_TMPDIR for 0 - flux exec -r 1 sh -c "test \"\$FLUX_TMPDIR\" != \"$FLUX_TMPDIR\"" +test_expect_success 'flux exec does not pass FLUX_URI' ' + # Ensure FLUX_URI for rank 1 doesn not equal FLUX_URI for 0 + flux exec -r 1 sh -c "test \"\$FLUX_URI\" != \"$FLUX_URI\"" ' test_expect_success 'flux exec passes cwd' ' diff --git a/t/t1005-cmddriver.t b/t/t1005-cmddriver.t index 4a34e3595af9..1b97d1ab0273 100755 --- a/t/t1005-cmddriver.t +++ b/t/t1005-cmddriver.t @@ -29,11 +29,6 @@ test_expect_success 'flux --broker-path sets FLUX_BROKER_PATH' ' | egrep "^FLUX_BROKER_PATH=/xyz/flux-broker$" ' -test_expect_success 'flux --tmpdr sets FLUX_TMPDIR' ' - flux -F --tmpdir /xyz /usr/bin/printenv \ - | egrep "^FLUX_TMPDIR=/xyz$" -' - test_expect_success 'flux --uri sets FLUX_URI' ' flux -F --uri xyz://foo /usr/bin/printenv \ | egrep "^FLUX_URI=xyz://foo$" @@ -44,18 +39,6 @@ test_expect_success 'flux --trace-handle sets FLUX_HANDLE_TRACE=1' ' | egrep "^FLUX_HANDLE_TRACE=1$" ' -test_expect_success 'flux --uri works for reconstructed default uri' ' - flux -F --uri local://$FLUX_TMPDIR comms info -' - -test_expect_success 'flux --uri works for uri with NULL path' ' - flux -F --uri local:// comms info -' - -test_expect_success 'flux --uri works for uri with whitespace path' ' - flux -F --uri "local:// " comms info -' - # ENOENT test_expect_success 'flux --uri fails for unknown connector' ' test_must_fail flux -F --uri 'noexist://' comms info @@ -81,11 +64,11 @@ test_expect_success 'flux env runs argument' " | grep ${FLUX_BUILD_DIR}/src/connectors " test_expect_success 'flux env passes cmddriver options' ' - flux -F --tmpdir /xyz env | grep "^FLUX_TMPDIR=/xyz" + flux -F --uri foo://xyz env | grep "^FLUX_URI=foo://xyz" ' test_expect_success 'flux env passes cmddriver option to argument' " - flux -F --tmpdir /xyx env sh -c 'echo \$FLUX_TMPDIR' \ - | grep ^/xyx$ + flux -F --uri foo://xyx env sh -c 'echo \$FLUX_URI' \ + | grep ^foo://xyx$ " # push /foo twice onto PYTHONPATH -- ensure it is leftmost position: test_expect_success 'cmddriver pushes dup path elements onto front of PATH' "