Merge pull request #5818 from garlick/housekeeping

job-manager: add support for housekeeping scripts with partial release of resources

configure.ac

@@ -599,6 +599,8 @@ AC_CONFIG_FILES( \
   etc/flux-hostlist.pc \
   etc/flux-taskmap.pc \
   etc/flux.service \
+  etc/[email protected] \
+  src/cmd/flux-run-housekeeping \
   doc/Makefile \
   doc/test/Makefile \
   t/Makefile \

doc/Makefile.am

@@ -55,7 +55,8 @@ MAN1_FILES_PRIMARY = \
 	man1/flux-cancel.1 \
 	man1/flux-watch.1 \
 	man1/flux-update.1 \
-	man1/flux-hostlist.1
+	man1/flux-hostlist.1 \
+	man1/flux-housekeeping.1
 
 # These files are generated as clones of a primary page.
 # Sphinx handles this automatically if declared in the conf.py
@@ -383,6 +384,7 @@ if ENABLE_DOCS
 man_MANS = $(MAN1_FILES) $(MAN3_FILES) $(MAN5_FILES) $(MAN7_FILES)
 $(RST_FILES): \
 	man1/common/resources.rst \
+	man1/common/experimental.rst \
 	man1/common/job-param-additional.rst \
 	man1/common/job-param-batch.rst \
 	man1/common/job-param-common.rst \
@@ -400,10 +402,13 @@ $(RST_FILES): \
 	man1/common/job-other-run.rst \
 	man1/common/job-shell-options.rst \
 	man3/common/resources.rst \
+	man3/common/experimental.rst \
 	man3/common/json_pack.rst \
 	man3/common/json_unpack.rst \
 	man5/common/resources.rst \
-	man7/common/resources.rst
+	man5/common/experimental.rst \
+	man7/common/resources.rst \
+	man7/common/experimental.rst
 endif
 
 SUFFIXES = .rst .1 .3 .5 .7
@@ -472,6 +477,7 @@ EXTRA_DIST = \
 	$(RST_FILES) \
 	man1/index.rst \
 	man1/common/resources.rst \
+	man1/common/experimental.rst \
 	man1/common/job-param-additional.rst \
 	man1/common/job-param-batch.rst \
 	man1/common/job-param-common.rst \
@@ -489,13 +495,16 @@ EXTRA_DIST = \
 	man1/common/job-other-run.rst \
 	man1/common/job-shell-options.rst \
 	man3/common/resources.rst \
+	man3/common/experimental.rst \
 	man3/index.rst \
 	man3/common/json_pack.rst \
 	man3/common/json_unpack.rst \
 	man5/common/resources.rst \
+	man5/common/experimental.rst \
 	man5/index.rst \
 	man7/index.rst \
 	man7/common/resources.rst \
+	man7/common/experimental.rst \
 	man7/flux-undocumented.rst
 
 CLEANFILES = \

doc/man1/common/experimental.rst

@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.

doc/man1/common/resources.rst

@@ -1,3 +1,5 @@
 Flux: http://flux-framework.org
 
 Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues

doc/man1/flux-housekeeping.rst

@@ -0,0 +1,137 @@
+====================
+flux-housekeeping(1)
+====================
+
+
+SYNOPSIS
+========
+
+| **flux** **housekeeping** **list** [*-n*] [*-o FORMAT*]
+| **flux** **housekeeping** **kill** [*--all*] [*-j JOBID*] [*-t HOSTS|RANKS*] [*-s SIGNUM*]
+
+
+DESCRIPTION
+===========
+
+.. program:: flux housekeeping
+
+The `EXPERIMENTAL`_ housekeeping service provides similar functionality to
+a job epilog, with a few advantages
+
+ - Housekeeping runs after the job, which is then allowed to exit CLEANUP
+   state and become inactive once resources are released.
+ - While housekeeping is running, the scheduler still thinks resources are
+   allocated to the job, and will not allocate resources to other jobs.
+ - Housekeeping supports partial release of resources back to the scheduler,
+   such that a subset of stuck nodes do not hold up other nodes from
+   being returned to service.
+
+The :program:`flux housekeeping` command is used to interact with the
+housekeeping service. It supports listing the resources currently executing
+housekeeping actions and a command to forcibly terminate actions on a per-job
+or per-node basis.
+
+
+COMMANDS
+========
+
+list
+----
+
+.. program:: flux housekeeping list
+
+:program:`flux housekeeping list` lists active housekeeping tasks by jobid.
+
+
+.. option:: -o, --format=FORMAT
+
+  Customize the output format (See the `OUTPUT FORMAT`_ section below).
+
+.. option:: -n, --no-header
+
+  Suppress header from output.
+
+kill
+----
+
+.. program:: flux housekeeping kill
+
+:program:`flux housekeeping kill` can be used to terminate active housekeeping
+tasks. Housekeeping may be terminated by jobid, a set of targets such as
+broker ranks or hostnames, or all housekeeping may be terminated via the
+:option:`--all` option.
+
+.. option:: -s, --signal=SIGNUM
+
+  Send signal SIGNUM instead of SIGTERM.
+
+.. option:: -t, --targets=RANK|HOSTS
+
+  Target a specific set of ranks or hosts.
+
+.. option:: -j, --jobid=JOBID
+
+  Target a specific job by JOBID. Without ``--targets`` this will kill all
+  housekeeping tasks for the specified job.
+
+.. option:: --all
+
+  Target all housekeeping tasks for all jobs.
+
+OUTPUT FORMAT
+=============
+
+The :option:`--format` option can be used to specify an output format using
+Python's string format syntax or a defined format by name. For a list of
+built-in and configured formats use :option:`-o help`.
+
+The following field names can be specified for
+:command:`flux housekeeping list`:
+
+**id**
+   The jobid that triggered housekeeping
+
+**runtime**
+   The time since this housekeeping task started
+
+**nnodes**
+   A synonym for **allocated.nnodes**
+
+**ranks**
+   A synonym for **allocated.ranks**
+
+**nodelist**
+   A synonym for **allocated.nodelist**
+
+**allocated.nnodes**
+   The number of nodes still allocated to this housekeeping task.
+
+**allocated.ranks**
+   The list of broker ranks still allocated to this housekeeping task.
+
+**allocated.ranks**
+   The list of nodes still allocated to this housekeeping task.
+
+**pending.nnodes**
+   The number of nodes that still need to complete housekeeping.
+
+**pending.ranks**
+   The list of broker ranks that still need to complete housekeeping.
+
+**pending.ranks**
+   The list of nodes that still need to complete housekeeping.
+
+EXPERIMENTAL
+============
+
+.. include:: common/experimental.rst
+
+RESOURCES
+=========
+
+.. include:: common/resources.rst
+
+SEE ALSO
+========
+
+:man5:`flux-config-job-manager`

doc/man3/common/experimental.rst

@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.

doc/man3/common/resources.rst

@@ -1,3 +1,5 @@
 Flux: http://flux-framework.org
 
 Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues

doc/man5/common/experimental.rst

@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.

doc/man5/common/resources.rst

@@ -1,3 +1,5 @@
 Flux: http://flux-framework.org
 
 Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues

doc/man5/flux-config-job-manager.rst

@@ -28,6 +28,20 @@ plugins
    Each directive follows the format defined in the :ref:`plugin_directive`
    section.
 
+housekeeping
+   (optional) Table of configuration for the job-manager housekeeping
+   service. The housekeeping service is an `EXPERIMENTAL`_ alternative for
+   handling administrative job epilog workloads. If enabled, resources are
+   released by jobs to housekeeping, which runs a command or a systemd unit
+   and releases resources to the scheduler on completion. See configuration
+   details in the :ref:`housekeeping` section.
+
+   Note: The housekeeping script runs as the instance owner (e.g. "flux").
+   On a real system, "command" is configured to "imp run housekeeping",
+   and the IMP is configured to launch the flux-housekeeping systemd
+   service as root. (See :man5:`flux-config-security-imp` for details
+   on configuring :command:`flux imp run`).
+
 
 .. _plugin_directive:
 
@@ -50,6 +64,28 @@ conf
    (optional) An object, valid with ``load`` only, that defines a configuration
    table to pass to the loaded plugin.
 
+.. _housekeeping:
+
+HOUSEKEEPING
+============
+
+command
+  (optional) An array of strings specifying the housekeeping command. Either
+  ``command`` or ``use-systemd-unit`` must be specified.
+
+use-systemd-unit
+  (optional) A boolean value indicating whether to run the flux-housekeeping
+  systemd unit to handle housekeeping, rather than a specific command.
+  Either ``use-systemd-unit`` or ``command`` must be specified.
+
+release-after
+  (optional) A string specified in Flux Standard Duration (FSD). If unset,
+  resources for a given job are not released until all execution targets for
+  a given job have completed housekeeping. If set to ``0``, resources are
+  released as each target completes. Otherwise, a timer is started when the
+  first execution target for a given job completes, and all resources that
+  have completed housekeeping when the timer fires are released. Following
+  that, resources are released as each execution target completes.
 
 EXAMPLE
 =======
@@ -73,6 +109,15 @@ EXAMPLE
       }
    ]
 
+   [job-manager.housekeeping]
+   use-systemd-unit = true
+   release-after = "1m"
+
+
+EXPERIMENTAL
+============
+
+.. include:: common/experimental.rst
 
 RESOURCES
 =========

doc/man7/common/experimental.rst

@@ -0,0 +1,4 @@
+Experimental Flux features and interfaces are made available for evaluation
+only and may change or be removed without notice.
+
+Feedback is welcome.  Please use the flux-core project Github issue tracker.

doc/man7/common/resources.rst

@@ -1,3 +1,5 @@
 Flux: http://flux-framework.org
 
 Flux RFC: https://flux-framework.readthedocs.io/projects/flux-rfc
+
+Issue Tracker: https://github.com/flux-framework/flux-core/issues

doc/manpages.py

@@ -70,6 +70,7 @@
     ('man1/flux-watch', 'flux-watch', 'monitor one or more Flux jobs', [author], 1),
     ('man1/flux-update', 'flux-update', 'update active Flux jobs', [author], 1),
     ('man1/flux-hostlist', 'flux-hostlist', 'fetch, combine, and manipulate Flux hostlists', [author], 1),
+    ('man1/flux-housekeeping', 'flux-housekeeping', 'list and terminate housekeeping tasks', [author], 1),
     ('man3/flux_attr_get', 'flux_attr_set', 'get/set Flux broker attributes', [author], 3),
     ('man3/flux_attr_get', 'flux_attr_get', 'get/set Flux broker attributes', [author], 3),
     ('man3/flux_aux_set', 'flux_aux_get', 'get/set auxiliary handle data', [author], 3),

etc/Makefile.am

@@ -1,5 +1,7 @@
 #if HAVE_SYSTEMD
-systemdsystemunit_DATA = flux.service
+systemdsystemunit_DATA = \
+	flux.service \
+	[email protected]
 #endif
 
 tmpfilesdir = $(prefix)/lib/tmpfiles.d

etc/completions/flux.pre

@@ -2014,6 +2014,46 @@ _flux_broker()
     return 0
 }
 
+# flux-housekeeping(1) completions
+_flux_housekeeping()
+{
+    local cmd=$1
+    local subcmds="list kill"
+    local split=false
+    list_OPTS="\
+        -o --format= \
+        -n --no-header \
+    "
+    kill_OPTS="\
+        -s --signal= \
+        -t --targets= \
+    "
+    _flux_split_longopt && split=true
+    case $prev in
+        --format | -!(-*)o)
+            _flux_complete_format_name flux housekeeping list
+            return
+            ;;
+    esac
+    if [[ $cmd != "housekeeping" ]]; then
+        if [[ $cur != -* ]]; then
+            if _flux_contains_word ${cmd} "kill"; then
+                hk_active=$(flux housekeeping list -no $(_flux_id_fmt $cur))
+                COMPREPLY=( $(compgen -W "${hk_active}" -- "$cur") )
+                return 0
+            fi
+        fi
+        var="${cmd//-/_}_OPTS"
+        COMPREPLY=( $(compgen -W "${!var}" -- "$cur") )
+        if [[ "${COMPREPLY[@]}" == *= ]]; then
+            # no space if suggestions ends with '='
+            compopt -o nospace
+        fi
+    else
+        COMPREPLY=( $(compgen -W "${subcmds}" -- "$cur") )
+    fi
+}
+
 _flux_core()
 {
     local cur prev cmd subcmd matched
@@ -2136,6 +2176,9 @@ _flux_core()
         compopt -o default
         COMPREPLY=()
         ;;
+    housekeeping)
+        _flux_housekeeping $subcmd
+        ;;
     -*)
         COMPREPLY=( $(compgen -W "${FLUX_OPTS}" -- "$cur") )
         ;;