Merge pull request flux-framework#3999 from grondo/flux-uri

add FluxURIResolver Python class and flux-uri command for job URI discovery

doc/Makefile.am

@@ -33,7 +33,8 @@ MAN1_FILES_PRIMARY = \
 	man1/flux-version.1 \
 	man1/flux-jobs.1 \
 	man1/flux-shell.1 \
-	man1/flux-jobtap.1
+	man1/flux-jobtap.1 \
+	man1/flux-uri.1
 
 # These files are generated as clones of a primary page.
 # Sphinx handles this automatically if declared in the conf.py
@@ -321,6 +322,7 @@ stderr_devnull_0 = >/dev/null 2>&1
 
 $(MAN_FILES): conf.py $(RST_FILES)
 	$(sphinx_man) \
+	PYTHONPATH=$(PYTHONPATH):$(abs_srcdir) \
 	SPHINX_BUILDDIR=$(abs_builddir) $(PYTHON) \
 		-m sphinx $(sphinx_verbose_flags) -b man $(srcdir) ./man \
 		$(STDERR_DEVNULL)
@@ -333,12 +335,14 @@ $(MAN_FILES): conf.py $(RST_FILES)
 .PHONY: html
 html: conf.py $(RST_FILES)
 	$(sphinx_html) \
+	PYTHONPATH=$(PYTHONPATH):$(abs_srcdir) \
 	SPHINX_BUILDDIR=$(abs_builddir) $(PYTHON) \
 		-m sphinx $(sphinx_verbose_flags) -b html $(srcdir) ./html \
 		$(STDERR_DEVNULL)
 
 EXTRA_DIST = \
 	conf.py \
+	domainrefs.py \
 	index.rst \
 	$(RST_FILES) \
 	man1/NODESET.rst \

doc/conf.py

@@ -46,9 +46,29 @@
 
 extensions = [
     'sphinx.ext.intersphinx',
-    'sphinx.ext.napoleon'
+    'sphinx.ext.napoleon',
+    'domainrefs'
 ]
 
+domainrefs = {
+    'man1': {
+        'text': "%s(1)",
+        'url': "../man1/%s.html"
+    },
+    'man3': {
+        'text': "%s(3)",
+        'url': "../man3/%s.html"
+    },
+    'man5': {
+        'text': "%s(5)",
+        'url': "../man5/%s.html"
+    },
+    'man7': {
+        'text': "%s(7)",
+        'url': "../man7/%s.html"
+    }
+}
+
 # Disable "smartquotes" to avoid things such as turning long-options
 #  "--" into en-dash in html output, which won't make much sense for
 #  manpages.
@@ -127,6 +147,7 @@ def setup(app):
     ('man1/flux-hwloc', 'flux-hwloc', 'Control/query resource-hwloc service', [author], 1),
     ('man1/flux-jobs', 'flux-jobs', 'list jobs submitted to Flux', [author], 1),
     ('man1/flux-jobtap', 'flux-jobtap', 'List, remove, and load job-manager plugins', [author], 1),
+    ('man1/flux-uri', 'flux-uri', 'resolve Flux URIs', [author], 1),
     ('man1/flux-keygen', 'flux-keygen', 'generate keys for Flux security', [author], 1),
     ('man1/flux-kvs', 'flux-kvs', 'Flux key-value store utility', [author], 1),
     ('man1/flux-logger', 'flux-logger', 'create a Flux log entry', [author], 1),

doc/domainrefs.py

@@ -0,0 +1,72 @@
+"""
+Originally from:
+
+  https://github.com/mitogen-hq/mitogen/blob/master/docs/domainrefs.py
+
+Copyright 2021, the Mitogen authors
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+"""
+import functools
+import re
+
+import docutils.nodes
+import docutils.utils
+
+
+CUSTOM_RE = re.compile('(.*) <(.*)>')
+
+
+def role(config, role, rawtext, text, lineno, inliner, options={}, content=[]):
+    template = 'https://docs.ansible.com/ansible/latest/modules/%s_module.html'
+
+    match = CUSTOM_RE.match(text)
+    if match:  # "custom text <real link>"
+        title = match.group(1)
+        text = match.group(2)
+    elif text.startswith('~'):  # brief
+        text = text[1:]
+        title = config.get('brief', '%s') % (
+            docutils.utils.unescape(text),
+        )
+    else:
+        title = config.get('text', '%s') % (
+            docutils.utils.unescape(text),
+        )
+
+    node = docutils.nodes.reference(
+        rawsource=rawtext,
+        text=title,
+        refuri=config['url'] % (text,),
+        **options
+    )
+
+    return [node], []
+
+
+def setup(app):
+    for name, info in app.config._raw_config['domainrefs'].items():
+        app.add_role(name, functools.partial(role, info))

doc/man1/flux-uri.rst

@@ -0,0 +1,147 @@
+===========
+flux-uri(1)
+===========
+
+
+SYNOPSIS
+========
+
+**flux** *uri* [OPTIONS] *TARGET*
+
+DESCRIPTION
+===========
+
+Connections to Flux are established via a Uniform Resource Indicator
+(URI) which is passed to the :man3:`flux_open` API call. These *native*
+URIs indicate the "connector" which will be used to establish the
+connection, and are typically either *local*, with a  ``local`` URI
+scheme, or *remote*, with a ``ssh`` URI scheme. These URIs are considered
+fully-resolved, native Flux URIs.
+
+Processes running within a Flux instance will have the ``FLUX_URI``
+environment variable set to a native URI which :man3:`flux_open` will
+use by default, with fallback to a compiled-in native URI for the system
+instance of Flux. Therefore, there is usually no need to specify a URI when
+connecting to the enclosing instance. However, connecting to a *different*
+Flux instance will require discovery of the fully-resolved URI for that
+instance.
+
+**flux uri** attempts to resolve its *TARGET* argument to a native local
+or remote URI. The *TARGET* is itself a URI which specifies the method
+to use in URI resolution via the scheme part, with the path and query
+parts passed to a plugin which implements the resolution method.
+
+As a convenience, if *TARGET* is specified with no scheme, then the scheme
+is assumed to be ``jobid``.  This allows ``flux uri`` to be used to look
+up the URI for a Flux instance running as a job in the current enclosing
+instance with:
+
+::
+
+   $ flux uri JOBID
+
+Depending on the *TARGET* URI scheme and corresponding plugin, specific
+query arguments int *TARGET* may be supported. However, as a convenience,
+all *TARGET* URIs support the special query arguments ``local`` or
+``remote`` to force the resulting URI into a local (``local://``) or remote
+(``ssh://``) form. For example:
+
+::
+
+   $ flux uri JOBID?local
+
+would return the ``local://`` URI for *JOBID* (if the URI can be resolved).
+
+A list of supported URI schemes will be listed at the bottom of
+``flux uri --help`` message. For a description of the URI resolver schemes
+included with Flux, see the URI SCHEMES and EXAMPLES sections below.
+
+OPTIONS
+=======
+
+**-remote**
+   Return the *remote* (``ssh://``)  equivalent of the resolved URI.
+
+**--local**
+   Return the *local* (``local://``) equivalent of the resulved URI.
+   Warning: the resulting URI may be invalid for the current system
+   if the network host specified by an ``ssh`` URI is not the current
+   host.
+
+URI SCHEMES
+===========
+
+The following URI schemes are included by default:
+
+jobid:ID[/ID...]
+   This scheme attempts to get the URI for a Flux instance running as a
+   job in the current enclosing instance. This is the assumed scheme if no
+   ``scheme:`` is provided in *TARGET* passed to ``flux uri``, so the
+   ``jobid:`` prefix is optional. A hierarchy of Flux jobids is supported,
+   so ``f1234/f3456`` will resolve the URI for job ``f3456`` running in
+   job ``f1234`` in the current instance.
+
+pid:PID
+  This scheme attempts to read the ``FLUX_URI`` value from the process id
+  *PID* using ``/proc/PID/environ``. If *PID* refers to a ``flux-broker``,
+  then the scheme reads ``FLUX_URI`` from the broker's initial program or
+  another child process since ``FLUX_URI`` in the broker's environment
+  would refer to *its* parent (or may not be set at all in the case of a
+  test instance started with ``flux start --test-size=N``).
+
+slurm:JOBID
+  This scheme makes a best-effort to resolve the URI of a Flux instance
+  launched under Slurm. It invokes ``srun`` to run ``scontrol listpids``
+  on the first node of the job, and then uses the ``pid`` resolver until
+  it finds a valid ``FLUX_URI``.
+
+
+EXAMPLES
+========
+
+To get the URI of a job in the current instance in its ``local://`` form:
+
+::
+
+   $ flux uri --local ƒN8Pz2xVu
+   local:///tmp/flux-zbVtVg/jobtmp-0-ƒN8Pz2xVu/flux-59uf5w/local-0
+
+or
+
+::
+
+   $ flux uri ƒN8Pz2xVu?local
+   local:///tmp/flux-zbVtVg/jobtmp-0-ƒN8Pz2xVu/flux-59uf5w/local-0
+
+
+Get the URI of a nested job:
+
+::
+
+   $ flux uri ƒqxxTiZBM/ƒr2XFWP?local
+   local:///tmp/flux-zbVtVg/jobtmp-0-ƒqxxTiZBM/flux-EPgSwk/local-0
+
+.. note::
+   With  the ``jobid`` resolver, ``?local`` only needs to be placed on
+   the last component of the jobid "path" or hierarchy. This will resolve
+   each URI in turn as a local URI.
+
+Get the URI of a local flux-broker
+
+::
+
+   $ flux uri pid:$(pidof -s flux-broker)
+   local:///tmp/flux-sLuBkZ/local-0
+
+Get the URI for a Flux instance running as a Slurm job:
+
+::
+
+   $ flux uri slurm:7843494
+   ssh://cluster42/var/tmp/user/flux-MpnytT/local-0
+
+
+RESOURCES
+=========
+
+Github: http://github.com/flux-framework

doc/man1/index.rst

@@ -26,5 +26,6 @@ man1
    flux-proxy
    flux-start
    flux-shell
+   flux-uri
    flux-version
    flux

src/bindings/python/flux/Makefile.am

@@ -9,6 +9,7 @@ nobase_fluxpy_PYTHON = \
 	future.py \
 	memoized_property.py \
 	debugged.py \
+	importer.py \
 	core/__init__.py \
 	core/watchers.py \
 	core/inner.py \
@@ -39,6 +40,11 @@ nobase_fluxpy_PYTHON = \
 	hostlist.py \
 	idset.py \
 	progress.py \
+	uri/uri.py \
+	uri/__init__.py \
+	uri/resolvers/jobid.py \
+	uri/resolvers/pid.py \
+	uri/resolvers/slurm.py \
 	utils/parsedatetime/__init__.py \
 	utils/parsedatetime/parsedatetime.py \
 	utils/parsedatetime/warns.py \

src/bindings/python/flux/importer.py

@@ -0,0 +1,56 @@
+###############################################################
+# Copyright 2021 Lawrence Livermore National Security, LLC
+# (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+#
+# This file is part of the Flux resource manager framework.
+# For details, see https://github.com/flux-framework.
+#
+# SPDX-License-Identifier: LGPL-3.0
+###############################################################
+
+import sys
+import os
+import pkgutil
+import importlib
+
+
+def import_plugins_pkg(ns_pkg):
+    """Import all modules found in the namespace package ``ns_pkg``"""
+    return {
+        name: importlib.import_module(f"{ns_pkg.__name__}.{name}")
+        for finder, name, ispkg in pkgutil.iter_modules(ns_pkg.__path__)
+    }
+
+
+def import_plugins(pkg_name, pluginpath=None):
+    """Load plugins from a namespace package and optional additional paths
+
+    A plugin in pluginpath with the same name as an existing plugin will
+    take precedence
+    """
+    if pluginpath is not None:
+        sys.path[1:1] = pluginpath
+
+    try:
+        #  Load 'pkg_name' as a namespace plugin
+        pkg = importlib.import_module(pkg_name)
+        plugins = import_plugins_pkg(pkg)
+    except ModuleNotFoundError:
+        return []
+
+    if pluginpath is not None:
+        #  Undo any added pluginpath elements.
+        for path in pluginpath:
+            sys.path.remove(path)
+
+    return plugins
+
+
+def import_path(file_path):
+    """Import a module directly from file_path"""
+
+    module_name = os.path.basename(file_path).rstrip(".py")
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module