Merge pull request #2578 from garlick/boot_conf

broker: new format for [bootstrap] configuration

configure.ac

@@ -489,6 +489,7 @@ AC_CONFIG_FILES( \
   doc/Makefile \
   doc/man1/Makefile \
   doc/man3/Makefile \
+  doc/man5/Makefile \
   doc/man7/Makefile \
   doc/test/Makefile \
   t/Makefile \

doc/Makefile.am

@@ -1 +1 @@
-SUBDIRS = man1 man3 man7 test
+SUBDIRS = man1 man3 man5 man7 test

doc/man5/COPYRIGHT.adoc

@@ -0,0 +1,4 @@
+Copyright 2014 Lawrence Livermore National Security, LLC
+and Flux developers.
+
+SPDX-License-Identifier: LGPL-3.0

doc/man5/Makefile.am

@@ -0,0 +1,27 @@
+MAN5_FILES = \
+	flux-config-bootstrap.5
+
+ADOC_FILES  = $(MAN5_FILES:%.7=%.adoc)
+XML_FILES   = $(MAN5_FILES:%.7=%.xml)
+
+if ENABLE_DOCS
+dist_man_MANS = $(MAN5_FILES)
+$(MAN5_FILES): COPYRIGHT.adoc
+endif
+
+SUFFIXES = .adoc .5
+
+STDERR_DEVNULL = $(stderr_devnull_$(V))
+stderr_devnull_ =  $(stderr_devnull_$(AM_DEFAULT_VERBOSITY))
+stderr_devnull_0 = 2>/dev/null
+
+.adoc.5:
+	$(AM_V_GEN)$(ADOC) --attribute mansource=$(PACKAGE_NAME) \
+	    --attribute manversion=$(PACKAGE_VERSION) \
+	    --attribute manmanual="Flux Miscellaneous Reference" \
+	    --destination-dir $(builddir) \
+	    --doctype manpage $(ADOC_FORMAT_OPT) manpage $< $(STDERR_DEVNULL)
+
+EXTRA_DIST = $(ADOC_FILES) COPYRIGHT.adoc
+
+CLEANFILES = $(MAN5_FILES) $(XML_FILES)

doc/man5/flux-config-bootstrap.adoc

@@ -0,0 +1,121 @@
+flux-config-bootstrap(5)
+========================
+:doctype: manpage
+
+
+NAME
+----
+flux-config-bootstrap - configure Flux instance bootstrap
+
+
+DESCRIPTION
+-----------
+
+The broker discovers the size of the Flux instance, the broker's rank,
+and overlay network wireup information either dynamically using a PMI
+service, such as when being launched by Flux or another resource manager,
+or statically using the `bootstrap` section of the Flux configuration,
+such as when being launched by systemd.
+
+The default bootstrap mode is PMI.  The Flux systemd unit file forces the
+broker to use the config method by specifying `--setattr=boot.method=config`
+on the broker command line.
+
+CONFIG FILES
+------------
+
+Flux uses the TOML configuration file format.  The broker normally
+parses `/etc/flux/conf.d/*.toml`.  The actual path is dependent on
+`configure` command line options used to build flux, and can be
+overridden with the FLUX_CONF_DIR environment variable.
+
+The `bootstrap` section is a TOML table containing the following
+keys.  Each node in a cluster is expected to bootstrap from an
+identical config file.
+
+KEYWORDS
+--------
+
+default_port::
+(optional) The value is an integer port number that is substituted
+for the token `%p` in the other keys.
+
+default_bind::
+(optional) The value is a ZeroMQ endpoint URI that is used for host
+entries that do not explicitly set a bind address.  The tokens
+`%p` and `%h` are replaced with the default port and the host
+for the current host entry.
+
+default_connect::
+(optional) The value is a ZeroMQ endpoint URI that is used for host
+entries that do not explicitly set a connect address.  The tokens
+`%p` and `%h` are replaced with the default port and the host
+for the current host entry.
+
+hosts::
+(optional) A rank-ordered array of host entries.  Each host entry is
+a TOML table containing at minimum the `host` key.  The broker determines
+its rank by matching its hostname in the hosts array and taking the array
+index.  An empty or missing hosts array implies a standalone (single
+broker) instance.  The entry for a broker with downstream peers must
+either assign the `bind` key to a ZeroMQ endpoint URI, or the `default_bind`
+URI described above is used.  The entry for a broker with downstream peers
+must also either assign the `connect` key to a ZeroMQ endpoint URI, or
+the `default_connect` URI described above is used.  The same `%h` and `%p`
+substitutions work here as well.
+
+COMPACT HOSTS
+-------------
+
+Since it would be tedious to repeat host entries for every compute node in
+a large cluster, the `hosts` array may be abbreviated using bracketed
+"idset" notation in `host` keys.
+
+An idset is an unordered set of non-negative integers that may be expressed
+as a comma-separated list including hyphenated ranges.  For example
+the set 0, 1, 2, 3, 4, 18, 20 may be represented as "0-4,18,20".
+
+A `host` key may include one or more bracketed idsets.  For example,
+"foo[0-1023]" represents the hosts "foo0, foo1, ..., foo1023", or
+"rack[0-1]node[0-1]" represents the hosts "rack0node0, rack0node1,
+rack1node0, rack1node1".
+
+EXAMPLE
+-------
+
+....
+[bootstrap]
+
+default_port = 8050
+default_bind = "tcp://en0:%p"
+default_connect = "tcp://e%h:%p"
+
+hosts = [
+    {
+        host="fluke0",
+        bind="tcp://en4:9001",
+        connect="tcp://fluke-mgmt:9001"
+    },
+    { host = "fluke[1-1023]" },
+]
+....
+
+
+AUTHOR
+------
+This page is maintained by the Flux community.
+
+
+RESOURCES
+---------
+Github: <http://github.com/flux-framework>
+
+
+COPYRIGHT
+---------
+include::COPYRIGHT.adoc[]
+
+
+SEE ALSO
+--------
+flux-getattr(1), flux_attr_get(3)

doc/test/spell.en.pws

@@ -466,3 +466,7 @@ FSD
 enqueues
 cpus
 gpu
+idsets
+systemd
+toml
+unordered

etc/Makefile.am

@@ -2,7 +2,8 @@
 systemdsystemunit_DATA = flux.service
 #endif
 
-dist_fluxcf_DATA = conf.d/boot.toml
+dist_fluxcf_DATA = \
+	conf.d/bootstrap.toml
 
 noinst_DATA = \
 	flux/curve

etc/conf.d/bootstrap.toml

@@ -0,0 +1,9 @@
+#
+# This bootstrap configuration is for a single-node
+# (i.e. workstation) Flux system configuration.
+#
+# See flux-config-bootstrap(5) for more information on this
+# configuration section.
+#
+
+[bootstrap]

etc/flux.service.in

@@ -3,7 +3,7 @@ Description=Flux message broker
 
 [Service]
 Environment=FLUX_USERDB_OPTIONS=--default-rolemask=user
-ExecStart=@X_BINDIR@/flux broker -Srundir=@X_RUNSTATEDIR@/flux -Sboot.method=config sleep inf
+ExecStart=@X_BINDIR@/flux broker -Srundir=@X_RUNSTATEDIR@/flux -Slocal-uri=local://@X_RUNSTATEDIR@/flux/local -Sboot.method=config sleep inf
 User=flux
 Group=flux
 RuntimeDirectory=flux

src/broker/Makefile.am

@@ -76,7 +76,8 @@ TESTS = test_shutdown.t \
 	test_service.t \
 	test_reduce.t \
 	test_liblist.t \
-	test_pmiutil.t
+	test_pmiutil.t \
+	test_boot_config.t
 
 test_ldadd = \
 	$(builddir)/libbroker.la \
@@ -127,3 +128,7 @@ test_liblist_t_LDADD = $(test_ldadd)
 test_pmiutil_t_SOURCES = test/pmiutil.c
 test_pmiutil_t_CPPFLAGS = $(test_cppflags)
 test_pmiutil_t_LDADD = $(test_ldadd)
+
+test_boot_config_t_SOURCES = test/boot_config.c
+test_boot_config_t_CPPFLAGS = $(test_cppflags)
+test_boot_config_t_LDADD = $(test_ldadd)