From 1c0af465ebe8a90e7e002444d505aa6c01681772 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Fri, 1 Feb 2019 16:09:17 -0800 Subject: [PATCH] remove wreck, jsc The wreck exec system is worthless, remove it along with associated commands, tests, and support code. Since libjsc doesn't work without wreck, it is removed as well. Fixes #1984 Closes #1947 Closes #1618 Closes #1595 Closes #1593 Closes #1534 Closes #1468 Closes #1443 Closes #1438 Closes #1419 Closes #1410 Closes #1407 Closes #1393 Closes #915 Closes #894 Closes #866 Closes #833 Closes #774 Closes #772 Closes #335 Closes #249 --- README.md | 2 - configure.ac | 2 - doc/man1/Makefile.am | 7 +- doc/man1/flux-start.adoc | 6 - doc/man1/flux-submit.adoc | 55 - doc/man1/flux-wreck.adoc | 177 -- doc/man1/flux-wreckrun.adoc | 250 -- doc/man1/wreck-extra-options.adoc | 81 - doc/man1/wreck-options.adoc | 85 - doc/test/spell.en.pws | 3 - etc/rc1 | 4 - etc/rc3 | 3 +- src/bindings/lua/Makefile.am | 16 +- src/bindings/lua/README | 4 - src/bindings/lua/flux/lustache.lua | 27 - src/bindings/lua/flux/lustache/context.lua | 66 - src/bindings/lua/flux/lustache/renderer.lua | 374 --- src/bindings/lua/flux/lustache/scanner.lua | 57 - src/bindings/lua/wreck.lua | 844 ------ src/bindings/lua/wreck/io.lua | 369 --- src/bindings/python/flux/Makefile.am | 1 - src/bindings/python/flux/__init__.py | 2 +- src/bindings/python/flux/jsc.py | 89 - src/broker/broker.c | 4 - src/cmd/.gitignore | 1 - src/cmd/Makefile.am | 4 - src/cmd/flux-hostlist | 72 +- src/cmd/flux-jstat.c | 270 -- src/cmd/flux-submit | 35 - src/cmd/flux-wreck | 909 ------- src/cmd/flux-wreckrun | 362 --- src/cmd/flux.c | 5 - src/common/Makefile.am | 2 - src/common/core.h | 1 - src/common/libflux-core.map | 1 - src/common/libflux/Makefile.am | 4 - src/common/libflux/conf.c | 4 - src/common/libjsc/Makefile.am | 18 - src/common/libjsc/README.md | 180 -- src/common/libjsc/jstatctl.c | 862 ------ src/common/libjsc/jstatctl.h | 136 - src/include/flux/core.h | 1 - src/modules/Makefile.am | 1 - src/modules/wreck/.gitignore | 1 - src/modules/wreck/Makefile.am | 97 - src/modules/wreck/job.c | 939 ------- src/modules/wreck/lua.d/01-env.lua | 57 - src/modules/wreck/lua.d/cuda_devices.lua | 75 - src/modules/wreck/lua.d/epilog.lua | 21 - src/modules/wreck/lua.d/input.lua | 202 -- src/modules/wreck/lua.d/intel_mpi.lua | 24 - src/modules/wreck/lua.d/mvapich.lua | 31 - src/modules/wreck/lua.d/openmpi.lua | 16 - src/modules/wreck/lua.d/output.lua | 126 - src/modules/wreck/lua.d/pmi-mapping.lua | 38 - src/modules/wreck/lua.d/spectrum.lua | 49 - src/modules/wreck/lua.d/timeout.lua | 61 - src/modules/wreck/luastack.c | 455 ---- src/modules/wreck/luastack.h | 95 - src/modules/wreck/rcalc.c | 390 --- src/modules/wreck/rcalc.h | 54 - src/modules/wreck/test/wreck_job.c | 257 -- src/modules/wreck/wreck_job.c | 247 -- src/modules/wreck/wreck_job.h | 85 - src/modules/wreck/wrexecd.c | 2619 ------------------- src/test/Makefile.am | 2 - src/test/cppcheck.sh | 1 - src/test/soak-workload.sh | 38 - src/test/soak.sh | 119 - t/Makefile.am | 46 - t/issues/t0704-mpirun-rank.sh | 3 - t/python/t0008-jsc.py | 97 - t/t1999-wreck-rcalc.t | 30 - t/t2000-wreck-dummy-sched.t | 63 - t/t2000-wreck-env.t | 51 - t/t2000-wreck-epilog.t | 69 - t/t2000-wreck-nokz.t | 106 - t/t2000-wreck.t | 596 ----- t/t2001-jsc.t | 267 -- t/t2002-pmi.t | 173 -- t/t2003-recurse.t | 84 - t/t2006-joblog.t | 40 - t/t2009-hostlist.t | 19 +- t/t2100-aggregate.t | 2 +- t/t3000-mpi-basic.t | 48 - t/t3001-mpi-personalities.t | 50 - t/t4000-issues-test-driver.t | 3 +- t/valgrind/workload.d/wreck | 8 - t/wreck/input/1r1c.json | 1 - t/wreck/input/1r2c.json | 1 - t/wreck/input/4r4c4r1c.json | 9 - t/wreck/input/8r1c.json | 9 - t/wreck/output/1r1c.1.err | 0 t/wreck/output/1r1c.1.expected | 3 - t/wreck/output/1r2c.1.err | 0 t/wreck/output/1r2c.1.expected | 3 - t/wreck/output/1r2c.2.err | 0 t/wreck/output/1r2c.2.expected | 3 - t/wreck/output/4r4c4r1c.16.err | 0 t/wreck/output/4r4c4r1c.16.expected | 10 - t/wreck/output/4r4c4r1c.20.err | 0 t/wreck/output/4r4c4r1c.20.expected | 10 - t/wreck/output/4r4c4r1c.4.err | 0 t/wreck/output/4r4c4r1c.4.expected | 10 - t/wreck/output/4r4c4r1c.8.err | 0 t/wreck/output/4r4c4r1c.8.expected | 10 - t/wreck/output/4r4c4r1c.9.err | 0 t/wreck/output/4r4c4r1c.9.expected | 10 - t/wreck/output/8r1c.4.err | 0 t/wreck/output/8r1c.4.expected | 10 - t/wreck/output/8r1c.8.err | 0 t/wreck/output/8r1c.8.expected | 10 - t/wreck/output/8r1c.9.err | 1 - t/wreck/output/8r1c.9.expected | 1 - t/wreck/rcalc.c | 65 - t/wreck/sched-dummy.c | 27 - 116 files changed, 13 insertions(+), 13430 deletions(-) delete mode 100644 doc/man1/flux-submit.adoc delete mode 100644 doc/man1/flux-wreck.adoc delete mode 100644 doc/man1/flux-wreckrun.adoc delete mode 100644 doc/man1/wreck-extra-options.adoc delete mode 100644 doc/man1/wreck-options.adoc delete mode 100644 src/bindings/lua/flux/lustache.lua delete mode 100644 src/bindings/lua/flux/lustache/context.lua delete mode 100644 src/bindings/lua/flux/lustache/renderer.lua delete mode 100644 src/bindings/lua/flux/lustache/scanner.lua delete mode 100644 src/bindings/lua/wreck.lua delete mode 100644 src/bindings/lua/wreck/io.lua delete mode 100644 src/bindings/python/flux/jsc.py delete mode 100644 src/cmd/flux-jstat.c delete mode 100755 src/cmd/flux-submit delete mode 100755 src/cmd/flux-wreck delete mode 100755 src/cmd/flux-wreckrun delete mode 100644 src/common/libjsc/Makefile.am delete mode 100644 src/common/libjsc/README.md delete mode 100644 src/common/libjsc/jstatctl.c delete mode 100644 src/common/libjsc/jstatctl.h delete mode 100644 src/modules/wreck/.gitignore delete mode 100644 src/modules/wreck/Makefile.am delete mode 100644 src/modules/wreck/job.c delete mode 100644 src/modules/wreck/lua.d/01-env.lua delete mode 100644 src/modules/wreck/lua.d/cuda_devices.lua delete mode 100644 src/modules/wreck/lua.d/epilog.lua delete mode 100644 src/modules/wreck/lua.d/input.lua delete mode 100644 src/modules/wreck/lua.d/intel_mpi.lua delete mode 100644 src/modules/wreck/lua.d/mvapich.lua delete mode 100644 src/modules/wreck/lua.d/openmpi.lua delete mode 100644 src/modules/wreck/lua.d/output.lua delete mode 100644 src/modules/wreck/lua.d/pmi-mapping.lua delete mode 100644 src/modules/wreck/lua.d/spectrum.lua delete mode 100644 src/modules/wreck/lua.d/timeout.lua delete mode 100644 src/modules/wreck/luastack.c delete mode 100644 src/modules/wreck/luastack.h delete mode 100644 src/modules/wreck/rcalc.c delete mode 100644 src/modules/wreck/rcalc.h delete mode 100644 src/modules/wreck/test/wreck_job.c delete mode 100644 src/modules/wreck/wreck_job.c delete mode 100644 src/modules/wreck/wreck_job.h delete mode 100644 src/modules/wreck/wrexecd.c delete mode 100755 src/test/soak-workload.sh delete mode 100755 src/test/soak.sh delete mode 100755 t/issues/t0704-mpirun-rank.sh delete mode 100755 t/python/t0008-jsc.py delete mode 100755 t/t1999-wreck-rcalc.t delete mode 100755 t/t2000-wreck-dummy-sched.t delete mode 100755 t/t2000-wreck-env.t delete mode 100755 t/t2000-wreck-epilog.t delete mode 100755 t/t2000-wreck-nokz.t delete mode 100755 t/t2000-wreck.t delete mode 100755 t/t2001-jsc.t delete mode 100755 t/t2002-pmi.t delete mode 100755 t/t2003-recurse.t delete mode 100755 t/t2006-joblog.t delete mode 100755 t/t3000-mpi-basic.t delete mode 100755 t/t3001-mpi-personalities.t delete mode 100755 t/valgrind/workload.d/wreck delete mode 100644 t/wreck/input/1r1c.json delete mode 100644 t/wreck/input/1r2c.json delete mode 100644 t/wreck/input/4r4c4r1c.json delete mode 100644 t/wreck/input/8r1c.json delete mode 100644 t/wreck/output/1r1c.1.err delete mode 100644 t/wreck/output/1r1c.1.expected delete mode 100644 t/wreck/output/1r2c.1.err delete mode 100644 t/wreck/output/1r2c.1.expected delete mode 100644 t/wreck/output/1r2c.2.err delete mode 100644 t/wreck/output/1r2c.2.expected delete mode 100644 t/wreck/output/4r4c4r1c.16.err delete mode 100644 t/wreck/output/4r4c4r1c.16.expected delete mode 100644 t/wreck/output/4r4c4r1c.20.err delete mode 100644 t/wreck/output/4r4c4r1c.20.expected delete mode 100644 t/wreck/output/4r4c4r1c.4.err delete mode 100644 t/wreck/output/4r4c4r1c.4.expected delete mode 100644 t/wreck/output/4r4c4r1c.8.err delete mode 100644 t/wreck/output/4r4c4r1c.8.expected delete mode 100644 t/wreck/output/4r4c4r1c.9.err delete mode 100644 t/wreck/output/4r4c4r1c.9.expected delete mode 100644 t/wreck/output/8r1c.4.err delete mode 100644 t/wreck/output/8r1c.4.expected delete mode 100644 t/wreck/output/8r1c.8.err delete mode 100644 t/wreck/output/8r1c.8.expected delete mode 100644 t/wreck/output/8r1c.9.err delete mode 100644 t/wreck/output/8r1c.9.expected delete mode 100644 t/wreck/rcalc.c delete mode 100644 t/wreck/sched-dummy.c diff --git a/README.md b/README.md index 917fa3f07eef..966bcd819303 100644 --- a/README.md +++ b/README.md @@ -139,8 +139,6 @@ Common commands from flux-core: start bootstrap a local Flux instance submit submit job requests to a scheduler user Flux user database client - wreck Flux wreck convenience utilities - wreckrun Flux utility for remote execution ``` Most of these have UNIX manual pages as `flux-(1)`, diff --git a/configure.ac b/configure.ac index 5b79843a44a8..b89f8e4859e4 100644 --- a/configure.ac +++ b/configure.ac @@ -394,7 +394,6 @@ AC_CONFIG_FILES( \ src/common/libflux/version.h \ src/common/libkvs/Makefile \ src/common/libkz/Makefile \ - src/common/libjsc/Makefile \ src/common/libjob/Makefile \ src/common/libzio/Makefile \ src/common/libsubprocess/Makefile \ @@ -422,7 +421,6 @@ AC_CONFIG_FILES( \ src/modules/kvs-watch/Makefile \ src/modules/content-sqlite/Makefile \ src/modules/barrier/Makefile \ - src/modules/wreck/Makefile \ src/modules/cron/Makefile \ src/modules/aggregator/Makefile \ src/modules/pymod/Makefile \ diff --git a/doc/man1/Makefile.am b/doc/man1/Makefile.am index 7df0973c7fcf..b2a8d7f60fb9 100644 --- a/doc/man1/Makefile.am +++ b/doc/man1/Makefile.am @@ -8,9 +8,6 @@ MAN1_FILES_PRIMARY = \ flux-keygen.1 \ flux-logger.1 \ flux-ping.1 \ - flux-submit.1 \ - flux-wreckrun.1 \ - flux-wreck.1 \ flux-start.1 \ flux-module.1 \ flux-exec.1 \ @@ -60,8 +57,6 @@ stderr_devnull_0 = 2>/dev/null EXTRA_DIST = \ $(ADOC_FILES) \ COPYRIGHT.adoc \ - NODESET.adoc \ - wreck-options.adoc \ - wreck-extra-options.adoc + NODESET.adoc CLEANFILES = $(MAN1_FILES) $(XML_FILES) diff --git a/doc/man1/flux-start.adoc b/doc/man1/flux-start.adoc index c1d5e1c38d1a..aff655b74bcc 100644 --- a/doc/man1/flux-start.adoc +++ b/doc/man1/flux-start.adoc @@ -83,12 +83,6 @@ shell as the initial program: srun --pty -N8 flux start -Launch an 8-way Flux instance under Flux, with /bin/true as -the initial program: - - flux wreckrun -N8 flux start /bin/true - - AUTHOR ------ This page is maintained by the Flux community. diff --git a/doc/man1/flux-submit.adoc b/doc/man1/flux-submit.adoc deleted file mode 100644 index ea86afcd6a90..000000000000 --- a/doc/man1/flux-submit.adoc +++ /dev/null @@ -1,55 +0,0 @@ -// flux-help-description: submit job requests to a scheduler -FLUX-SUBMIT(1) -============== -:doctype: manpage - - -NAME ----- -flux-submit - Flux utility for submitting job requests to a scheduler - - -SYNOPSIS --------- -[verse] -'flux submit' [-n ] [-N ] [-t ] - [-o|--options='OPTIONS'] - [-O|--output='FILENAME'] [-E|--error='FILENAME'] - [-i|--input='HOW'] ['COMMANDS'...] - - -DESCRIPTION ------------ - -'flux submit' is the front-end command for submitting jobs to -a Flux comms session for eventual execution by a Flux scheduler. -Once a job has been scheduled for execution, it will be executed -using the WRECK remote execution prototype. - -See flux-wreckrun(1) for more information about execution -method. - -OPTIONS -------- - -include::wreck-options.adoc[] - -include::wreck-extra-options.adoc[] - -AUTHOR ------- -This page is maintained by the Flux community. - - -RESOURCES ---------- -Github: - - -COPYRIGHT ---------- -include::COPYRIGHT.adoc[] - -SEE ALSO --------- -flux-wreckrun(1), flux-wreck(1) diff --git a/doc/man1/flux-wreck.adoc b/doc/man1/flux-wreck.adoc deleted file mode 100644 index 50709c526b9b..000000000000 --- a/doc/man1/flux-wreck.adoc +++ /dev/null @@ -1,177 +0,0 @@ -// flux-help-description: Flux wreck convenience utilities -FLUX-WRECK(1) -============= -:doctype: manpage - - -NAME ----- -flux-wreck - Flux wreck facility convenience utility - - -SYNOPSIS --------- -*flux* *wreck* 'COMMAND' ['OPTIONS'] - - -DESCRIPTION ------------ -'flux wreck' is a convenience utility that operates on jobs in the -WRECK remote execution prototype. It can be used to gather information -about running and completed jobs directly from the Flux kvs. - -flux-wreck(1) operates via a series of 'COMMANDS', described below: - -COMMANDS --------- -*help* 'cmd':: -Print help. If 'cmd' is provided, print help for that sub-command. - -*ls* [-n, --max=COUNT] [JOBIDS...]:: -Display a list of wreck jobs currently in kvs, along with their current -states. If '-n, --max' option is provided, then display at most 'COUNT' -jobs (default: 25). If an optional list of 'JOBIDS' is provided on the -command line, then display only those jobs. - -*uri* [-n, --max=COUNT] [-b, --bare] [JOBIDS...]:: -Display a list of wreck jobs currently in kvs, with abbreviated job info, -and a URI that can be used to contact the rank 0 broker if the job is a -Flux instance. The field is blank for other types of jobs. -If '-n, --max' option is provided, then display at most 'COUNT' -jobs (default: 25). If '-b, --bare' option is provided, display only the -URI for a single job so that it can be parsed by scripts. If an optional -list of 'JOBIDS' is provided on the command line, then display only those jobs. - -*attach* [--status] [--label-io] 'jobid':: -Attach to output of a running or completed job. If input was not previously -connected, also attach to stdin. With '--status', also report job status -after dumping output. With '--label-io' label lines of output with taskid. - -*status* 'jobid':: -Return the status of running or completed job 'jobid'. Prints the current -job status, and if exited, the exit codes of all tasks. This command will -exit with the highest exit code of all tasks. - -*cancel* 'jobid':: -Make a request to the scheduler to cancel a pending job 'jobid'. Only supported -if a job scheduler module is loaded. - -*kill* [--signal=N] 'jobid':: -Send SIGTERM to running job 'jobid'. If '--signal' is used, send signal 'N' -where 'N' may be a signal number or name such as 'SIGKILL'. - -*setenv* [VAR=VALUE|all]:: -Export environment variables to all future jobs. The 'flux wreck setenv', -'getenv' and 'unsetenv' commands control the global job environment which -is applied to all jobs run by the wreck system. With 'flux wreck setenv all' -the entire environment is exported to the global job environment, inherited -by all future jobs. This may help reduce the size of job environment records -stored in the KVS if there are only small environment differences from -run to run. 'flux wreck setenv VAR=VALUE' may also be used to export a -single environment variable to all jobs. - -*getenv* VAR:: -Get the value of an environment variable set in the global job environment. - -*unsetenv* VAR:: -Unset environment variable 'VAR' in the global job environment. - -*setopt* 'option[=value]':: -Set global job option 'option' with optional value 'value' (the default value -is 'true'). All future jobs will inherit global job options as if they -were run with '-o option=value'. - -*getopt* '[option]':: -Display global job options. - -*timing* [-n, --max=COUNT] [JOBIDS...]:: -List timing information for lwj entries in Flux KVS. if '-n, --max' option is -provided, then display at most 'COUNT' jobs (default: 25). If an optional list -of 'JOBIDS' is provided on the command line, then display timing information -for only those jobs. This command lists four specific intervals in the wreck -job lifecycle: - 'STARTING'::: Time from LWJ entry creation until the job reaches the - "starting" state. The "starting" state is when the first - rexec daemon receives the request to launch tasks. - 'RUNNING'::: Time from "starting" to "running". The running state is - reached when all rexec daemons have forked and executed - all tasks. - 'COMPLETE'::: Time from "running" to "complete". This is the nominal time - that tasks of the job actually ran. - 'TOTAL'::: This is the total time from job creation to the "completed" state. - - -*kvs-path* 'jobid':: -Display the path in the Flux KVS to the specified 'jobid'. - -*last-jobid* [--kvs-path]:: -Return the current value of the 'lwj' sequence number, that is the -last jobid allocated to a submitted wreck job. With '--kvs-path', -report the path to job in kvs instead of jobid. Other jobs may be -started between the time the id is obtained and when it is displayed -in this command, so 'last-jobid' should not be relied upon during real -workloads. - -*purge* [OPTIONS...]:: -Purge old job entries from Flux KVS. Without any options, the 'purge' -subcommand will print the current number of entries in the 'lwj.' -kvs directory. Otherwise, 'flux wreck purge' walks the list of all -completed jobs in the Flux KVS and selects the oldest (in terms of -completion time) as candidates for removal. The number of jobs selected -is dependent on the options below: - ---verbose::: --v::: - Increase verbosity. - ---target-size='COUNT'::: --t 'COUNT'::: - Set the target number of job entries to keep to 'COUNT'. The number - of IDs to remove will be calculated based on target size, number of - current job entries, and any maximum set on command line. - ---remove::: --R::: - Perform an unlink of all selected job entries in KVS. This option is - only valid if supplied with '--target-size'. - ---max=N::: --m N::: - Set the maximum number of entries to remove on any one run. - - -*dumplog* [JOBID]:: -Dump the error log from the wreck job 'JOBID'. - -*exclude* [NODENAME]:: -Exclude a node from running future jobs. By default, jobs currently running -on the node and its resources are not killed. - --k, --kill::: - Kill all jobs to which the node is allocated. - -*include* [NODENAME]:: -Include a previously excluded node from scheduling. - -*sched-params* set 'item=val':: -*sched-params* get 'item':: -Set and get scheduler parameters at runtime. - -AUTHOR ------- -This page is maintained by the Flux community. - - -RESOURCES ---------- -Github: - - -COPYRIGHT ---------- -include::COPYRIGHT.adoc[] - - -SEE ALSO --------- -flux-wreckrun(1) diff --git a/doc/man1/flux-wreckrun.adoc b/doc/man1/flux-wreckrun.adoc deleted file mode 100644 index f3568cffb95f..000000000000 --- a/doc/man1/flux-wreckrun.adoc +++ /dev/null @@ -1,250 +0,0 @@ -// flux-help-include: yes -FLUX-WRECKRUN(1) -================ -:doctype: manpage - - -NAME ----- -flux-wreckrun - Flux utility for remote execution - - -SYNOPSIS --------- -[verse] -'flux wreckrun' [-n ] [-N ] [-t ] - [-l|--label-io] [-d|--detach] [-o|--options='OPTIONS'] - [-O|--output='FILENAME'] [-E|--error='FILENAME'] - [-i|--input='HOW'] [-I, --immediate] ['COMMANDS'...] - - -DESCRIPTION ------------ - -'flux wreckrun' is the front-end command for launching tasks within -a Flux comms session using the WRECK remote execution prototype. - -WRECK (Worthless Remote Execution using CMB KVS) is a prototype -implementation of remote execution for Flux which demonstrates -the usage of the Flux key-value store as the vehicle for transmitting -and storing information about lightweight jobs (LWJs). The basic -mode of operation is described in xref:wreck-operation[]. - -OPTIONS -------- - -include::wreck-options.adoc[] - ---detach:: --d:: - Detach immediately after issuing a run request and print new jobid - to stdout. - ---wait-until='state':: --w 'state':: - Ignore all stdio, and instead just wait until the job reaches - a state of 'state' (i.e. "starting", "running", or "complete") and - exit immediately. - ---immediate:: --I:: - Bypass scheduler and run job immediately. - - -include::wreck-extra-options.adoc[] - -OPERATION ----------- -[[wreck-operation]] - -Briefly, the method of operation for remote execution via WRECK -consists of the following steps: - -. The front-end command ('wreckrun') allocates a new job directory - entry in the kvs under the 'lwj' key. This operation is performed as - a 'job.create' request, which results in a new directory in the kvs, - such as 'lwj.'. The full directory path is dependent on - settings but can be determined via "flux wreck last-jobid -p". - Initially, a job state is 'reserved' and stored in the 'state' - directory, such as 'lwj..state'. -. The front-end command now fills in requisite and optional job - information under the 'lwj' jobid directory such as the command-line - to run, how many tasks to run on each rank, the job environment, - etc. -. The front-end command then issues a 'wrexec.run' event, which - is handled by the 'wrexec' module. The 'wrexec' module spawns - a daemon to handle the request for the jobid in question. The - 'wrexecd' daemon pulls information for the job in question - directly from the local kvs cache and spawns required tasks. -. Once the first task is started by rank 0, the job state is updated - to 'starting'. -. After starting tasks, all 'wrexecd' daemons synchronize on a - barrier, and the job state is updated to 'running'. -. As the tasks run, stdio is either stored in the KVS or sent directly - to a per-job service as messages (when '-o nokz' job option is set), - and output is optionally displayed to user or redirected as requested - by command line options. -. As tasks exit, their exit status is recorded in the kvs. -. After all tasks have exited, 'wrexecd' daemons synchronize again - and rank 0 updates the job state to 'completed'. - -This simple scheme offers a scalable and semi-flexible method -for launching tasks within a Flux comms session. In conjunction -with the Flux PMI library, even MPI applications may be launched -at scale via 'flux wreckrun'. See xref:wreckrun-examples[] for -examples. - -OUTPUT ENVIRONMENT VARIABLES ----------------------------- -[[wreckrun-output-environment-variables]] - -The following environment variables will be set in the environment of -tasks executed via 'flux wreckrun'. - -*FLUX_JOB_ID*:: -The job ID of the current job - -*FLUX_JOB_NNODES*:: -Number of nodes used in the current job - -*FLUX_NODE_ID*:: -The relative node ID of the current node - -*FLUX_JOB_SIZE*:: -The total number of tasks to be run in the current job - -*FLUX_LOCAL_RANKS*:: -Comma separated list of task ranks run on the current node - -*FLUX_TASK_RANK*:: -The relative task rank of the current task - -*FLUX_TASK_LOCAL_ID*:: -The relative task rank of the current task on the current node - - -EXAMPLES --------- -[[wreckrun-examples]] - -Once a Flux comms session is running, ensure the required modules -for 'wreck' are loaded: - ------------------------------- -$ flux module list | egrep 'resource|job|wrexec' -resource-hwloc 44673 498403A idle 0 -wrexec 42052 2CA4CAT idle 0 -job 28959 60947AA idle 0 -$ ------------------------------- - -To ensure wrexec is working properly, run hostname across -4 nodes of the Flux comms session: - ------------------------------- -$ flux wreckrun -l -N4 hostname -wreckrun: 0.000s: Sending LWJ request for 1 tasks (cmdline "hostname") -wreckrun: 0.007s: Registered jobid 1 -wreckrun: Allocating 4 tasks across 4 available nodes.. -wreckrun: tasks per node: node[0-3]: 1 -wreckrun: 0.009s: Sending run event -wreckrun: 0.012s: State = reserved -wreckrun: 0.020s: State = starting -wreckrun: 0.082s: State = running -0: hype349 -2: hype351 -3: hype352 -1: hype350 -wreckrun: 0.140s: State = complete -wreckrun: All tasks completed successfully. ------------------------------- - -'flux wreckrun' uses the Flux kvs for most of its functionality, -and thus information about lightweight jobs (LWJs) that are -executed will be stored in the 'lwj.' directory in the kvs. -To see what kind of information is stored there, use the -'flux kvs' utility: - ------------------------------- -$ flux kvs dir lwj.1 ------------------------------- - -.Running an MPI application with 'flux wreckrun' - -Launching an MPI application with 'flux wreckrun' requires that your -version of MPI supports PMI. In this example, MVAPICH2 is used: - ------------------------------- -$ mpicc -v 2>&1 | head -1 -mpicc for MVAPICH2 version 1.7 -$ mpicc -o mpi_hello mpi_hello.c ------------------------------- - -In order to run using the Flux PMI library, 'LD_LIBRARY_PATH' may need -to be adjusted at runtime. For example, to use Flux's libpmi.so -from the build directory: - ------------------------------- -$ export LD_LIBRARY_PATH=${flux_builddir}/src/lib/libpmi/.libs${LD_LIBRARY_PATH} ------------------------------- - -where '${flux_builddir}' is the path to the root of the flux-core -build directory. - -Now the MPI application should run via 'flux wreckrun': - ------------------------------- -$ flux wreckrun -N4 ./mpi_hello -wreckrun: 0.000s: Sending LWJ request for 1 tasks (cmdline "./mpi_hello") -wreckrun: 0.007s: Registered jobid 5 -wreckrun: Allocating 4 tasks across 4 available nodes.. -wreckrun: tasks per node: node[0-3]: 1 -wreckrun: 0.009s: Sending run event -wreckrun: 0.011s: State = reserved -wreckrun: 0.018s: State = starting -wreckrun: 0.028s: State = running -0: completed first barrier -0: completed MPI_Init in 0.039s. There are 4 tasks -0: completed MPI_Finalize -wreckrun: 2.182s: State = complete -wreckrun: All tasks completed successfully. ------------------------------- - -The use of Flux PMI can be verified by checking the KVS 'pmi' directory -for the LWJ in question: - ------------------------------- -$ flux kvs dir pmi.5 -pmi.5.hostname[2] = "-1463773063" -pmi.5.hostname[0] = "-1463773575" -pmi.5.hostname[1] = "-1463773319" -pmi.5.hostname[3] = "-1463772807" -pmi.5.uuid_2_5 = "dummy-entry" -pmi.5.uuid_0_5 = "8hiWSLCPRzqFYHNubktbCQ==" -pmi.5.uuid_1_5 = "dummy-entry" -pmi.5.uuid_3_5 = "dummy-entry" -pmi.5.pmi_epidkey_2 = "1901059" -pmi.5.pmi_epidkey_0 = "7209475" -pmi.5.pmi_epidkey_1 = "6816259" -pmi.5.pmi_epidkey_3 = "6750723" ------------------------------- - - -AUTHOR ------- -This page is maintained by the Flux community. - - -RESOURCES ---------- -Github: - - -COPYRIGHT ---------- -include::COPYRIGHT.adoc[] - - -SEE ALSO --------- -flux-wreck(1) diff --git a/doc/man1/wreck-extra-options.adoc b/doc/man1/wreck-extra-options.adoc deleted file mode 100644 index a6a5ccb38ef2..000000000000 --- a/doc/man1/wreck-extra-options.adoc +++ /dev/null @@ -1,81 +0,0 @@ -EXTRA OPTIONS -------------- -[[extra-options]] - -Extra options can be supplied on the `flux wreckrun` command via the -'--options' command line option. Multiple options should be separated -via commas. Currently available options include: - -'cpu-affinity[=per-task]':: - Set default CPU affinity of all tasks to the cores allocated - to the job. With 'cpu-affinity=per-task' distribute tasks - across allocated cores using 'hwloc_distrib(3)'. - -'gpubind=[per-task,off]':: - By default, if GPU devices are allocated to a job, the environment - variable CUDA_VISIBLE_DEVICES is set to the list of allocated - GPUs. The 'gpubind' option may be used to disable this behavior - '-o gpubind=off', or can request that the GPUs be distributed - among the local tasks with '-o gpubind=per-task'. - -'mpi':: - Select an MPI type being run if necessary. Some MPI "plugins" - are not enabled by default, and this option may be used to - enable them, e.g. '-o mpi=spectrum'. - -'nokz':: - Do not use KVS based "kz" streams for stdout and stderr streams - of this job. This option will bypass the KVS for job output by - using messages directly sent to a per-job service registered by - either wreckrun or the first wrexecd of the job. - -'kz':: - Force use of KVS based "kz" streams for stdout and stderr streams - for this job. This option may be used in place of '-o nokz=false' - when 'nokz' has been set as the default for all jobs via global - wreck options (i.e. if 'flux wreck getopt nokz' reports true) - -'stdio-commit-on-open':: - Commit to kvs on stdio open in each task. The default is to - delay commit. Without this option, stdio files for jobs will - not appear in KVS on all nodes until the next KVS commit. - The default behavior avoids a kvs commit storm as stdout and - stderr files are opened for every task. - -'stdio-commit-on-close':: - Commit to kvs for each stdio stream as it gets EOF. The default - is to delay the commit. Without this option, stdio files for - jobs will not appear to close until after the next natural - KVS commit. The default behavior avoids a kvs commit storm - as stdout and stderr files are closed from many tasks. This - option may need to be enabled if you need to detect immediately - when a stdout or stderr stream is closed from a task. - -'stdio-delay-commit':: - Disable commit to kvs for each line of output. The default - behavior is to call kvs_commit() after each line of output - from every task. If many tasks are writing multiple lines - of output and it is not necessary to see lines of output - as they are generated, it will speed up job execution to - enable this option. - -'commit-on-task-exit':: - Commit to the kvs for each task exit event. The default behavior - is to write the task exit status to the kvs as each task in - a lightweight job exits, but delay kvs commit. This avoids - a kvs commit storm when many tasks are exiting simultaneously. - This option may need to be enabled if knowlege of exit status or - fact that tasks have exited is required in real time. - -'stop-children-in-exec':: - Start tasks in a STOPPED state waiting for attach from a - debugger. This option is provided for use with parallel - debuggers such as TotalView. - -'no-pmi-server':: - Do not start simple pmi server. This option can be used to - not launch the pmi server if a non-MPI job is being run. - -'trace-pmi-server':: - Log simple pmi server protocol exchange. This option is used - for debugging. diff --git a/doc/man1/wreck-options.adoc b/doc/man1/wreck-options.adoc deleted file mode 100644 index 76c247fef156..000000000000 --- a/doc/man1/wreck-options.adoc +++ /dev/null @@ -1,85 +0,0 @@ - ---ntasks='ntasks':: --n 'ntasks':: - Request to run a total of 'ntasks' tasks. If '--ntasks' is used alone, - i.e. without '--nnodes', then 'flux wreckrun' will attempt to - distribute the 'ntasks' tasks across all nodes of the Flux comms - session. - ---cores-per-task='N':: --c 'N':: - Request N cores be allocated to each task. - ---gpus-per-task='N':: --g 'N':: - Request N GPUs be allocated to each task. - ---nnodes='nnodes':: --N 'nnodes':: - Request to run across 'nnodes' nodes in the current comms session. - ---tasks-per-node='tpn':: --t 'tpn':: - Set the number of tasks to run per node to 'tpn'. The default is - --tasks-per-node=1. This value is overridden by '--ntasks' if used. - ---label-io:: --l:: - Prepend stdout and stderr output lines with the task id to which - the output belongs. - ---output='FILENAME':: --O 'FILENAME':: - Duplicate stdout and stderr from tasks to a file or files. 'FILENAME' - is optionally a mustache template which expands the keys 'id', 'cmd' - and 'taskid'. (e.g. '--output=flux-{{id}}.out') - ---error='FILENAME':: --E 'FILENAME':: - Send stderr to a different location than stdout. - ---input='HOW':: --i 'HOW':: - Indicate how to deal with stdin for tasks. 'HOW' is a list of 'src:dst' - pairs where 'src' is a source 'FILENAME' which may optionally be a - mustache template as in `--output`, or the special term `stdin` to - indicate the stdin from a front end program, and 'dst' is a list of - taskids or `*` or `all` to indicate all tasks. The default is `stdin:*`. - If only one of 'src' or 'dst' is specified, a heuristic is used to - determine whether a list of tasks or an input file was meant. (e.g - `--input=file.in` will use `file.in` as input for all tasks, and - `--input=0` will send stdin to task 0 only. - ---walltime='N[SUFFIX]':: --T 'N[SUFFIX]':: - Set max job walltime to N seconds. Optional suffix may be 's' - for seconds (default), 'm' for minutes, 'h' for hours or 'd' - for days. N may be an arbitrary floating point number, but - will be rounded up to nearest second. - ---skip-env:: --S:: - Skip the export of the local environment to the job. - ---epilog='PATH':: --x 'PATH':: - Execute a script at 'PATH' after all tasks exit but before the - job state is set to to 'complete' (So the job will not complete - and the resources will not be released until the script ends) - ---postscript='PATH':: --p 'PATH':: - Similar to '--epilog', but execute 'PATH' after job state is - complete. - ---options='options,...':: --o 'options,...':: - Apply extended job options to the current execution. Examples of - these options are described in the xref:extra-options[]. - ---name='JOBNAME':: --J 'JOBNAME':: - Set an optional name to 'JOBNAME' for the submitted job. This name - will be saved in the KVS record for the job and displayed in place - of command name in `flux wreck ls` output. - diff --git a/doc/test/spell.en.pws b/doc/test/spell.en.pws index acdebc725f66..796ebb39523e 100644 --- a/doc/test/spell.en.pws +++ b/doc/test/spell.en.pws @@ -96,9 +96,6 @@ runtime src tpn uuid -wreckrun -wrexec -wrexecd ary baz EPGM diff --git a/etc/rc1 b/etc/rc1 index 8b53c82c470c..a7b49f25c0fe 100755 --- a/etc/rc1 +++ b/etc/rc1 @@ -11,14 +11,10 @@ flux module load -r all aggregator flux hwloc reload & pids="$pids $!" -flux module load -r all job flux module load -r 0 cron sync=hb flux module load -r 0 userdb ${FLUX_USERDB_OPTIONS} -# make flux wreckrun/submit -onokz the default (override w/ -o kz): -flux wreck setopt nokz - flux module load -r all job-ingest flux module load -r 0 job-manager diff --git a/etc/rc3 b/etc/rc3 index d98702365e31..189ebf3eb58d 100755 --- a/etc/rc3 +++ b/etc/rc3 @@ -16,13 +16,12 @@ flux module remove -r 0 job-manager flux module remove -r all job-ingest if PERSISTDIR=$(flux getattr persist-directory 2>/dev/null); then - flux wreck ls >${PERSISTDIR}/joblog 2>/dev/null || : + /bin/true; # XXX: nothing to persist? fi flux module remove -r 0 userdb flux module remove -r 0 cron -flux module remove -r all job flux module remove -r all aggregator flux module remove -r all barrier diff --git a/src/bindings/lua/Makefile.am b/src/bindings/lua/Makefile.am index 9a3b248f42c4..2cf346ab6cc3 100644 --- a/src/bindings/lua/Makefile.am +++ b/src/bindings/lua/Makefile.am @@ -10,28 +10,14 @@ AM_CPPFLAGS = \ fluxluadir = $(luadir)/flux fluxometerdir = $(luadir)/fluxometer -wreckluadir = $(luadir)/wreck -lustachedir = $(fluxluadir)/lustache fluxluaexecdir = $(luaexecdir)/flux -dist_lua_SCRIPTS = \ - wreck.lua - nobase_dist_lua_SCRIPTS = \ flux/timer.lua \ flux/alt_getopt.lua \ flux/posix.lua \ flux/base64.lua \ - flux/lustache.lua \ - flux/Subcommander.lua \ - wreck/io.lua - -# Required because install(1) doesn't seem to preserve -# full path when copying to directory: -dist_lustache_SCRIPTS = \ - flux/lustache/context.lua \ - flux/lustache/renderer.lua \ - flux/lustache/scanner.lua + flux/Subcommander.lua nobase_dist_fluxometer_SCRIPTS = \ Test/Builder/NoOutput.lua \ diff --git a/src/bindings/lua/README b/src/bindings/lua/README index 97637ba02a08..cea75e5e2aa6 100644 --- a/src/bindings/lua/README +++ b/src/bindings/lua/README @@ -64,10 +64,6 @@ First, a quick example to display the basics: end end - - For another example, see dlua/wreck.lua in-tree. It is a reimplementation - of wreck/wreckrun in pure Lua. - - ======================= 2. Flux API usage: diff --git a/src/bindings/lua/flux/lustache.lua b/src/bindings/lua/flux/lustache.lua deleted file mode 100644 index b85b1f5580cb..000000000000 --- a/src/bindings/lua/flux/lustache.lua +++ /dev/null @@ -1,27 +0,0 @@ --- lustache: Lua mustache template parsing. --- Copyright 2013 Olivine Labs, LLC --- MIT Licensed. - -local string_gmatch = string.gmatch - -function string.split(str, sep) - local out = {} - for m in string_gmatch(str, "[^"..sep.."]+") do out[#out+1] = m end - return out -end - -local lustache = { - name = "lustache", - version = "1.3-1", - renderer = require("flux.lustache.renderer"):new(), -} - -return setmetatable(lustache, { - __index = function(self, idx) - if self.renderer[idx] then return self.renderer[idx] end - end, - __newindex = function(self, idx, val) - if idx == "partials" then self.renderer.partials = val end - if idx == "tags" then self.renderer.tags = val end - end -}) diff --git a/src/bindings/lua/flux/lustache/context.lua b/src/bindings/lua/flux/lustache/context.lua deleted file mode 100644 index edb0d3573845..000000000000 --- a/src/bindings/lua/flux/lustache/context.lua +++ /dev/null @@ -1,66 +0,0 @@ -local string_find, string_split, tostring, type = - string.find, string.split, tostring, type - -local context = {} - -function context:clear_cache() - self.cache = {} -end - -function context:push(view) - return self:new(view, self) -end - -function context:lookup(name) - local value = self.cache[name] - - if not value then - if name == "." then - value = self.view - else - local context = self - - while context do - if string_find(name, ".") > 0 then - local names = string_split(name, ".") - local i = 0 - - value = context.view - - if(type(value)) == "number" then - value = tostring(value) - end - - while value and i < #names do - i = i + 1 - value = value[names[i]] - end - else - value = context.view[name] - end - - if value then - break - end - - context = context.parent - end - end - - self.cache[name] = value - end - - return value -end - -function context:new(view, parent) - local out = { - view = view, - parent = parent, - cache = {}, - magic = "1235123123", --ohgodwhy - } - return setmetatable(out, { __index = self }) -end - -return context diff --git a/src/bindings/lua/flux/lustache/renderer.lua b/src/bindings/lua/flux/lustache/renderer.lua deleted file mode 100644 index 593f506cd003..000000000000 --- a/src/bindings/lua/flux/lustache/renderer.lua +++ /dev/null @@ -1,374 +0,0 @@ -local Scanner = require "flux.lustache.scanner" -local Context = require "flux.lustache.context" - -local error, ipairs, loadstring, pairs, setmetatable, tostring, type = - error, ipairs, loadstring, pairs, setmetatable, tostring, type -local math_floor, math_max, string_find, string_gsub, string_split, string_sub, table_concat, table_insert, table_remove = - math.floor, math.max, string.find, string.gsub, string.split, string.sub, table.concat, table.insert, table.remove - -local patterns = { - white = "%s*", - space = "%s+", - nonSpace = "%S", - eq = "%s*=", - curly = "%s*}", - tag = "[#\\^/>{&=!]" -} - -local html_escape_characters = { - ["&"] = "&", - ["<"] = "<", - [">"] = ">", - ['"'] = """, - ["'"] = "'", - ["/"] = "/" -} - -local function is_array(array) - if type(array) ~= "table" then return false end - local max, n = 0, 0 - for k, _ in pairs(array) do - if not (type(k) == "number" and k > 0 and math_floor(k) == k) then - return false - end - max = math_max(max, k) - n = n + 1 - end - return n == max -end - --- Low-level function that compiles the given `tokens` into a --- function that accepts two arguments: a Context and a --- Renderer. - -local function compile_tokens(tokens, originalTemplate) - local subs = {} - - local function subrender(i, tokens) - if not subs[i] then - local fn = compile_tokens(tokens, originalTemplate) - subs[i] = function(ctx, rnd) return fn(ctx, rnd) end - end - return subs[i] - end - - local function render(ctx, rnd) - local buf = {} - local token, section - for i, token in ipairs(tokens) do - local t = token.type - buf[#buf+1] = - t == "#" and rnd:_section( - token, ctx, subrender(i, token.tokens), originalTemplate - ) or - t == "^" and rnd:_inverted( - token.value, ctx, subrender(i, token.tokens) - ) or - t == ">" and rnd:_partial(token.value, ctx, originalTemplate) or - (t == "{" or t == "&") and rnd:_name(token.value, ctx, false) or - t == "name" and rnd:_name(token.value, ctx, true) or - t == "text" and token.value or "" - end - return table_concat(buf) - end - return render -end - -local function escape_tags(tags) - - return { - string_gsub(tags[1], "%%", "%%%%").."%s*", - "%s*"..string_gsub(tags[2], "%%", "%%%%"), - } -end - -local function nest_tokens(tokens) - local tree = {} - local collector = tree - local sections = {} - local token, section - - for i,token in ipairs(tokens) do - if token.type == "#" or token.type == "^" then - token.tokens = {} - sections[#sections+1] = token - collector[#collector+1] = token - collector = token.tokens - elseif token.type == "/" then - if #sections == 0 then - error("Unopened section: "..token.value) - end - - -- Make sure there are no open sections when we're done - section = table_remove(sections, #sections) - - if not section.value == token.value then - error("Unclosed section: "..section.value) - end - - section.closingTagIndex = token.startIndex - - if #sections > 0 then - collector = sections[#sections].tokens - else - collector = tree - end - else - collector[#collector+1] = token - end - end - - section = table_remove(sections, #sections) - - if section then - error("Unclosed section: "..section.value) - end - - return tree -end - --- Combines the values of consecutive text tokens in the given `tokens` array --- to a single token. -local function squash_tokens(tokens) - local out, txt = {}, {} - local txtStartIndex, txtEndIndex - for _, v in ipairs(tokens) do - if v.type == "text" then - if #txt == 0 then - txtStartIndex = v.startIndex - end - txt[#txt+1] = v.value - txtEndIndex = v.endIndex - else - if #txt > 0 then - out[#out+1] = { type = "text", value = table_concat(txt), startIndex = txtStartIndex, endIndex = txtEndIndex } - txt = {} - end - out[#out+1] = v - end - end - if #txt > 0 then - out[#out+1] = { type = "text", value = table_concat(txt), startIndex = txtStartIndex, endIndex = txtEndIndex } - end - return out -end - -local function make_context(view) - if not view then return view end - return view.magic == "1235123123" and view or Context:new(view) -end - -local renderer = { } - -function renderer:clear_cache() - self.cache = {} - self.partial_cache = {} -end - -function renderer:compile(tokens, tags, originalTemplate) - tags = tags or self.tags - if type(tokens) == "string" then - tokens = self:parse(tokens, tags) - end - - local fn = compile_tokens(tokens, originalTemplate) - - return function(view) - return fn(make_context(view), self) - end -end - -function renderer:render(template, view, partials) - if type(self) == "string" then - error("Call mustache:render, not mustache.render!") - end - - if partials then - -- remember partial table - -- used for runtime lookup & compile later on - self.partials = partials - end - - if not template then - return "" - end - - local fn = self.cache[template] - - if not fn then - fn = self:compile(template, self.tags, template) - self.cache[template] = fn - end - - return fn(view) -end - -function renderer:_section(token, context, callback, originalTemplate) - local value = context:lookup(token.value) - - if type(value) == "table" then - if is_array(value) then - local buffer = "" - - for i,v in ipairs(value) do - buffer = buffer .. callback(context:push(v), self) - end - - return buffer - end - - return callback(context:push(value), self) - elseif type(value) == "function" then - local section_text = string_sub(originalTemplate, token.endIndex+1, token.closingTagIndex - 1) - - local scoped_render = function(template) - return self:render(template, context) - end - - return value(section_text, scoped_render) or "" - else - if value then - return callback(context, self) - end - end - - return "" -end - -function renderer:_inverted(name, context, callback) - local value = context:lookup(name) - - -- From the spec: inverted sections may render text once based on the - -- inverse value of the key. That is, they will be rendered if the key - -- doesn't exist, is false, or is an empty list. - - if value == nil or value == false or (is_array(value) and #value == 0) then - return callback(context, self) - end - - return "" -end - -function renderer:_partial(name, context, originalTemplate) - local fn = self.partial_cache[name] - - -- check if partial cache exists - if (not fn and self.partials) then - - local partial = self.partials[name] - if (not partial) then - return "" - end - - -- compile partial and store result in cache - fn = self:compile(partial, nil, originalTemplate) - self.partial_cache[name] = fn - end - return fn and fn(context, self) or "" -end - -function renderer:_name(name, context, escape) - local value = context:lookup(name) - - if type(value) == "function" then - value = value(context.view) - end - - local str = value == nil and "" or value - str = tostring(str) - - if escape then - return string_gsub(str, '[&<>"\'/]', function(s) return html_escape_characters[s] end) - end - - return str -end - --- Breaks up the given `template` string into a tree of token objects. If --- `tags` is given here it must be an array with two string values: the --- opening and closing tags used in the template (e.g. ["<%", "%>"]). Of --- course, the default is to use mustaches (i.e. Mustache.tags). -function renderer:parse(template, tags) - tags = tags or self.tags - local tag_patterns = escape_tags(tags) - local scanner = Scanner:new(template) - local tokens = {} -- token buffer - local spaces = {} -- indices of whitespace tokens on the current line - local has_tag = false -- is there a {{tag} on the current line? - local non_space = false -- is there a non-space char on the current line? - - -- Strips all whitespace tokens array for the current line if there was - -- a {{#tag}} on it and otherwise only space - - local type, value, chr - - while not scanner:eos() do - local start = scanner.pos - - value = scanner:scan_until(tag_patterns[1]) - - if value then - for i = 1, #value do - chr = string_sub(value,i,i) - - if string_find(chr, "%s+") then - spaces[#spaces+1] = #tokens - else - non_space = true - end - - tokens[#tokens+1] = { type = "text", value = chr, startIndex = start, endIndex = start } - start = start + 1 - end - end - - if not scanner:scan(tag_patterns[1]) then - break - end - - has_tag = true - type = scanner:scan(patterns.tag) or "name" - - scanner:scan(patterns.white) - - if type == "=" then - value = scanner:scan_until(patterns.eq) - scanner:scan(patterns.eq) - scanner:scan_until(tag_patterns[2]) - elseif type == "{" then - local close_pattern = "%s*}"..tags[2] - value = scanner:scan_until(close_pattern) - scanner:scan(patterns.curly) - scanner:scan_until(tag_patterns[2]) - else - value = scanner:scan_until(tag_patterns[2]) - end - - if not scanner:scan(tag_patterns[2]) then - error("Unclosed tag at " .. scanner.pos) - end - - tokens[#tokens+1] = { type = type, value = value, startIndex = start, endIndex = scanner.pos - 1 } - if type == "name" or type == "{" or type == "&" then - non_space = true --> what does this do? - end - - if type == "=" then - tags = string_split(value, patterns.space) - tag_patterns = escape_tags(tags) - end - end - - return nest_tokens(squash_tokens(tokens)) -end - -function renderer:new() - local out = { - cache = {}, - partial_cache = {}, - tags = {"{{", "}}"} - } - return setmetatable(out, { __index = self }) -end - -return renderer diff --git a/src/bindings/lua/flux/lustache/scanner.lua b/src/bindings/lua/flux/lustache/scanner.lua deleted file mode 100644 index 0673df112305..000000000000 --- a/src/bindings/lua/flux/lustache/scanner.lua +++ /dev/null @@ -1,57 +0,0 @@ -local string_find, string_match, string_sub = - string.find, string.match, string.sub - -local scanner = {} - --- Returns `true` if the tail is empty (end of string). -function scanner:eos() - return self.tail == "" -end - --- Tries to match the given regular expression at the current position. --- Returns the matched text if it can match, `null` otherwise. -function scanner:scan(pattern) - local match = string_match(self.tail, pattern) - - if match and string_find(self.tail, pattern) == 1 then - self.tail = string_sub(self.tail, #match + 1) - self.pos = self.pos + #match - - return match - end - -end - --- Skips all text until the given regular expression can be matched. Returns --- the skipped string, which is the entire tail of this scanner if no match --- can be made. -function scanner:scan_until(pattern) - - local match - local pos = string_find(self.tail, pattern) - - if pos == nil then - match = self.tail - self.pos = self.pos + #self.tail - self.tail = "" - elseif pos == 1 then - match = nil - else - match = string_sub(self.tail, 1, pos - 1) - self.tail = string_sub(self.tail, pos) - self.pos = self.pos + #match - end - - return match -end - -function scanner:new(str) - local out = { - str = str, - tail = str, - pos = 1 - } - return setmetatable(out, { __index = self } ) -end - -return scanner diff --git a/src/bindings/lua/wreck.lua b/src/bindings/lua/wreck.lua deleted file mode 100644 index d2fa3223192b..000000000000 --- a/src/bindings/lua/wreck.lua +++ /dev/null @@ -1,844 +0,0 @@ -------------------------------------------------------------- --- Copyright 2014 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - -local posix = require 'flux.posix' -local hostlist = require 'flux.hostlist' -local cpuset = require 'flux.affinity'.cpuset - -local wreck = {} -wreck.__index = wreck; - -local lwj_options = { --- ['ntasks'] = "Set total number of tasks to execute", - ['commit-on-task-exit'] = "Call kvs_commit for each task exit", - ['stdio-delay-commit'] = "Don't call kvs_commit for each line of output", - ['stdio-commit-on-open'] = "Commit to kvs on stdio open in each task", - ['stdio-commit-on-close'] = "Commit to kvs on stdio close in each task", - ['nokz'] = "Do not store job output in kvs", - ['kz'] = "Force job output to kvs", - ['stop-children-in-exec'] = "Start tasks in STOPPED state for debugger", - ['no-pmi-server'] = "Do not start simple-pmi server", - ['trace-pmi-server'] = "Log simple-pmi server protocol exchange", - ['cpu-affinity'] = "Set default cpu-affinity to assigned cores", - ['gpubind'] = "Control CUDA_VISIBLE_DEVICES [=per-task,off]", - ['mpi'] = "Set hint for type of MPI, e.g. -o mpi=spectrum " -} - -local default_opts = { - ['help'] = { char = 'h' }, - ['name'] = { char = 'J', arg = "NAME" }, - ['verbose'] = { char = 'v' }, - ['ntasks'] = { char = 'n', arg = "N" }, - ['gpus-per-task'] = { char = 'g', arg = "g" }, - ['cores-per-task'] = { char = 'c', arg = "N" }, - ['nnodes'] = { char = 'N', arg = "N" }, - ['tasks-per-node'] = - { char = 't', arg = "N" }, - ['walltime'] = { char = "T", arg = "SECONDS" }, - ['output'] = { char = "O", arg = "FILENAME" }, - ['error'] = { char = "E", arg = "FILENAME" }, - ['input'] = { char = "i", arg = "HOW" }, - ['label-io'] = { char = "l", }, - ['skip-env'] = { char = "S", }, - ['epilog'] = { char = "x", arg = "SCRIPT" }, - ['postscript'] = - { char = "p", arg = "SCRIPT" }, - ['options'] = { char = 'o', arg = "OPTIONS.." }, -} - -local function opt_table (w) - local o = {} - for x,t in pairs (default_opts) do - o[x] = t.char - end - for _,v in pairs (w.extra_options) do - o[v.name] = v.char - end - return o -end - -local function short_opts (w) - local s = "" - for x,t in pairs (default_opts) do - s = s .. t.char .. (t.arg and ":" or "") - end - for _,v in pairs (w.extra_options) do - s = s .. v.char .. (v.arg and ":" or "") - end - return s -end - -function wreck:say (...) - io.stderr:write (self.prog..": "..string.format (...)) -end - -function wreck:verbose (...) - if self.opts.v then - self:say (...) - end -end - -function wreck:die (...) - self:say (...) - os.exit (1) -end - -function wreck:usage() - io.stderr:write ("Usage: "..self.prog.." OPTIONS.. COMMANDS..\n") - io.stderr:write ([[ - -h, --help Display this message - -v, --verbose Be verbose - -N, --name=NAME Set an optional name for job to NAME - -n, --ntasks=N Request to run a total of N tasks - -c, --cores-per-task=N Request N cores per task - -g, --gpus-per-task=N Request N GPUs per task - -N, --nnodes=N Force number of nodes - -t, --tasks-per-node=N Force number of tasks per node - -o, --options=OPTION,... Set other options (See OTHER OPTIONS below) - -T, --walltime=N[SUFFIX] Set max job walltime to N seconds. Optional - suffix may be 's' for seconds (default), 'm' - for minutes, 'h' for hours or 'd' for days. - N may be an arbitrary floating point number, - but will be rounded up to nearest second. - -O, --output=FILENAME Duplicate stdout/stderr from tasks to a file or - files. FILENAME is optionally a mustache - template with keys such as id, cmd, taskid. - (e.g. --output=flux-{{id}}.out) - -i, --input=HOW Indicate how to deal with stdin for tasks. HOW - is a list of [src:]dst pairs separated by semi- - colon, where src is an optional src file - (default: stdin) and dst is a list of taskids - or "*" or "all". E.g. (-i0 sends stdin to task 0, - -i /dev/null:* closes all stdin, etc.) - -E, --error=FILENAME Send stderr to a different location than stdout. - -l, --labelio Prefix lines of output with task id - -S, --skip-env Skip export of environment to job - -x, --epilog=PATH Execute a script after all tasks exit but before - the job state is set to "complete" - -p, --postscript=PATH Execute a script after job state is "complete" -]]) - for _,v in pairs (self.extra_options) do - local optstr = v.name .. (v.arg and "="..v.arg or "") - io.stderr:write ( - string.format (" -%s, --%-20s %s\n", v.char, optstr, v.usage)) - end - io.stderr:write ("\nOTHER OPTIONS:\n") - for o,d in pairs (lwj_options) do - io.stderr:write (string.format (" %-26s %s\n", o, d)) - end -end - - -function wreck.get_filtered_env () - local env = posix.getenv() - env.HOSTNAME = nil - env.ENVIRONMENT = nil - for k,v in pairs (env) do - if k:match ("SLURM_") then env[k] = nil end - if k:match ("FLUX_URI") then env[k] = nil end - end - return (env) -end - -local function get_job_env (arg) - local f = arg.flux - local env = wreck.get_filtered_env () - local default_env = {} - if f then default_env = f:kvs_get ("lwj.environ") or {} end - for k,v in pairs (env) do - -- If same key=value is already in default env no need to - -- export it again, remove: - if default_env[k] == env[k] then env[k] = nil end - end - return (env) -end - -local function array_tonumber (t) - for i = 1, #t do - t[i] = tonumber (t[i]) - end - return t -end - - -local function job_kvspath (f, id) - assert (id, "Required argument id missing!") - local arg = { id } - if type (id) == "table" then - arg = id - end - local r, err = f:rpc ("job.kvspath", {ids = array_tonumber (arg) }) - if not r then error (err) end - if type (id) == "table" then return r.paths end - return r.paths [1] -end - ---- --- Return kvs path to job id `id` --- -local kvs_paths = {} -local function kvs_path_insert (id, path) - kvs_paths [id] = path -end -local function kvs_path (f, id) - if not kvs_paths[id] then - kvs_path_insert (id, job_kvspath (f, id)) - end - return kvs_paths [id] -end - -local function kvs_path_multi (f, ids) - local result = job_kvspath (f, ids) - for i,id in ipairs (ids) do - kvs_path_insert (id, result [id]) - end - return result -end - -function wreck:lwj_path (id) - assert (id, "missing required argument `id'") - if not self.lwj_paths[id] then - self.lwj_paths [id] = job_kvspath (self.flux, id) - end - return self.lwj_paths [id] -end - -function wreck:kvsdir (id) - if not self.kvsdirs[id] then - local f = self.flux - local d, err = f:kvsdir (self:lwj_path (id)) - if not d then return nil, err end - self.kvsdirs[id] = d - end - return self.kvsdirs[id] -end - -function wreck:add_options (opts) - for k,v in pairs (opts) do - if not type (v) == "table" then return nil, "Invalid parameter" end - - local char = v.char - if default_opts [v.name] then - return nil, "Can't override option '"..k.."'" - end - table.insert (self.extra_options, v) - end - return true -end - -function wreck:getopt (opt) - if not self.opts then return nil end - return self.opts [opt] -end - -local function parse_walltime (s) - local m = { s = 1, m = 60, h = 3600, d = 56400 } - local n, suffix = s:match ("^([0-9.]+)([HhMmDdSs]?)$") - if not tonumber (n) then - return nil, "Invalid duration '"..s.."'" - end - if suffix and m [suffix] then - n = (n * m [suffix]) - end - return math.ceil (n) -end - -function wreck:initial_job_options () - if not self.flux then - local err - self.flux, err = require 'flux'.new () - if not self.flux then - self:die ("Unable to open connection to flux broker: %s", err) - end - end - return self.flux:kvs_get ("lwj.options") or {} -end - --- Create output configuration table from wreck object state, -function wreck:output_file_config () - if not self.opts.O and not self.opts.E then return nil end - return { - files = { - stdout = self.opts.O, - stderr = self.opts.E, - }, - labelio = self.opts.l, - } -end - -function wreck:parse_cmdline (arg) - local getopt = require 'flux.alt_getopt' .get_opts - local s = short_opts (self) - local v = opt_table (self) - self.opts, self.optind = getopt (arg, s, v) - - if self.opts.h then - self:usage() - os.exit (0) - end - - if self.opts.o then - for opt in self.opts.o:gmatch ("[^,]+") do - if not lwj_options [opt:match ("([^=]+)")] then - return nil, string.format ("Unknown LWJ option '%s'\n", opt) - end - end - end - - if self.opts.T then - self.walltime, err = parse_walltime (self.opts.T) - if not self.walltime then - self:die ("Error: %s", err) - end - end - - if self.optind > #arg then - self:say ("Error: remote command required\n") - self:usage() - os.exit (1) - end - - self.nnodes = self.opts.N and tonumber (self.opts.N) - - if not self.opts.t then - self.opts.t = 1 - end - - -- If nnodes was provided but -n, --ntasks not set, then - -- set ntasks to nnodes. - if self.opts.N and not self.opts.n then - self.ntasks = self.nnodes * self.opts.t - else - self.ntasks = self.opts.n and tonumber (self.opts.n) or 1 - end - if self.opts.c then - self.ncores = self.opts.c * self.ntasks - else - self.ncores = self.ntasks - end - - if self.opts.g then - self.ngpus = self.opts.g * self.ntasks - end - - self.tasks_per_node = self.opts.t - - self.cmdline = {} - for i = self.optind, #arg do - table.insert (self.cmdline, arg[i]) - end - - if not self.flux then - self.flux = require 'flux'.new() - end - - self.job_options = self:initial_job_options () - if self.opts.o then - for entry in self.opts.o:gmatch ('[^,]+') do - local opt, val = entry, 1 - if entry:match ("=") then - opt, val = entry:match ("(.+)=(.+)") - if tonumber(val) == 0 or val == "false" or val == "no" then - val = false - end - end - -- kz/nokz override global value of eachother - if opt == "kz" and self.job_options.nokz then - self.job_options.nokz = false - end - if opt == "nokz" and self.job_options.kz then - self.job_options.kz = false - end - self.job_options[opt] = val - end - end - - self.output = self:output_file_config () - return true -end - -local function inputs_table_from_args (wreck, s) - -- Default is to broadcast stdin to all tasks: - if not s then return { { src = "stdin", dst = "*" } } end - - local inputs = {} - for m in s:gmatch ("[^;]+") do - local src,dst = m:match ("^([^:]-):?([^:]+)$") - if not src or not dst then - wreck:die ("Invalid argument to --input: %s\n", s) - end - -- A dst of "none" short circuits rest of entries - if dst == "none" then return inputs end - - -- Verify the validity of dst - if dst ~= "all" and dst ~= "*" then - local h, err = cpuset.new (dst) - if not h then - if src ~= "" then - wreck:die ("--input: invalid dst: %s\n", dst) - end - -- otherwise, assume only dst was provided - src = dst - dst = "*" - end - end - if src == "" or src == "-" then src = "stdin" end - table.insert (inputs, { src = src, dst = dst }) - end - -- Add fall through option - table.insert (inputs, { src = "/dev/null", dst = "*" }) - return inputs -end - -local function fixup_nnodes (wreck) - if not wreck.flux then return end - if not wreck.nnodes then - wreck.nnodes = math.min (wreck.ntasks, wreck.flux.size) - elseif wreck.nnodes > wreck.flux.size then - wreck:die ("Requested nodes (%d) exceeds available (%d)\n", - wreck.nnodes, wreck.flux.size) - end -end - -local function random_string () - local f = assert (io.open ("/dev/urandom", "rb")) - local d = f:read(8) - f:close() - local s = "" - for i = 1, d:len() do - s = s..string.format ("%02x", d:byte(i)) - end - return (s) -end - -function wreck:setup_ioservices () - -- Setup stderr/stdout ioservice ranks. Either flux-wreckrun - -- or nodeid 0 of the job will offer one or both of these services: - self.ioservices = {} - if self.job_options.nokz then - -- Default: register stdout and stderr service on this rank - local name = random_string () - self.ioservices = { - stdout = { - name = name..":out", - rank = self.flux.rank - }, - stderr = { - name = name..":err", - rank = self.flux.rank - } - } - if self.output then - -- Shunt stdio to nodeid 0 of the job if any output is being - -- redirected to files via the output.lua plugin: - local outfile = self.output.files.stdout - local errfile = self.output.files.stderr - if outfile then - self.ioservices.stdout.rank = -1 - end - if errfile or outfile then - self.ioservices.stderr.rank = -1 - end - end - end - return self.ioservices -end - - -function wreck:jobreq () - if not self.opts then return nil, "Error: cmdline not parsed" end - if self.fixup_nnodes then - fixup_nnodes (self) - end - local jobreq = { - name = self.opts.J, - nnodes = self.nnodes or 0, - ntasks = self.ntasks, - ncores = self.ncores, - cmdline = self.cmdline, - environ = self.opts.S and {} or get_job_env { flux = self.flux }, - cwd = posix.getcwd (), - walltime =self.walltime or 0, - ngpus = self.ngpus or 0, - - ["opts.nnodes"] = self.opts.N, - ["opts.ntasks"] = self.opts.n, - ["opts.cores-per-task"] = self.opts.c, - ["opts.gpus-per-task"] = self.opts.g, - ["opts.tasks-per-node"] = self.opts.t, - ["epilog.pre"] = self.opts.x, - ["epilog.post"] = self.opts.p, - } - jobreq.options = self.job_options - jobreq.output = self.output - jobreq ["input.config"] = inputs_table_from_args (self, self.opts.i) - - jobreq.ioservice = self:setup_ioservices () - - return jobreq -end - -local function send_job_request (self, name) - if not self.flux then - local f, err = require 'flux'.new () - if not f then return nil, err end - self.flux = f - end - local jobreq, err = self:jobreq () - if not jobreq then - return nil, err - end - local resp, err = self.flux:rpc (name, jobreq) - if not resp then - return nil, "flux.rpc: "..err - end - if resp.errnum then - return nil, name.." message failed with errnum="..resp.errnum - end - return resp -end - -function wreck:submit () - local resp, err = send_job_request (self, "job.submit") - if not resp then return nil, err end - if resp.state ~= "submitted" then - return nil, "job submission failure, job left in state="..resp.state - end - return resp.jobid, resp.kvs_path -end - -function wreck:createjob () - self.fixup_nnodes = true - local resp, err = send_job_request (self, "job.create") - if not resp then return nil, err end - -- - -- If reply contains a kvs path to this LWJ, pre-memoize the id - -- to kvs path mapping so we do not have to fetch it again. - if resp.kvs_path then - self.lwj_paths [resp.jobid] = resp.kvs_path - kvs_paths [resp.jobid] = resp.kvs_path - end - return resp.jobid, resp.kvs_path -end - -local function initialize_args (arg) - if arg.ntasks and arg.nnodes then return true end - local f = arg.flux - local lwj, err = f:kvsdir (kvs_path (f, arg.jobid)) - if not lwj then - return nil, "Error: "..err - end - arg.ntasks = lwj.ntasks - arg.nnodes = lwj.nnodes - return true -end - -function wreck.ioattach (arg) - local ioplex = require 'wreck.io' - local f = arg.flux - local rc, err = initialize_args (arg) - if not rc then return nil, err end - - local taskio, err = ioplex.create (arg) - if not taskio then return nil, err end - for i = 0, arg.ntasks - 1 do - for _,stream in pairs {"stdout", "stderr"} do - local rc, err = taskio:redirect (i, stream, stream) - end - end - taskio:start (arg.flux) - return taskio -end - -local logstream = {} -logstream.__index = logstream - -local function logstream_print_line (iow, line) - io.stderr:write (string.format ("rank%d: Error: %s\n", iow.id, line)) -end - -function logstream:dump () - for _, iow in pairs (self.watchers) do - local r, err = iow.kz:read () - while r and r.data and not r.eof do - logstream_print_line (iow, r.data) - r, err = iow.kz:read () - end - end -end - -function wreck.logstream (arg) - local l = {} - local f = arg.flux - if not f then return nil, "flux argument member required" end - local rc, err = initialize_args (arg) - if not rc then return nil, err end - if not arg.nnodes then return nil, "nnodes argument required" end - l.watchers = {} - for i = 0, arg.nnodes - 1 do - local key = kvs_path (f, arg.jobid)..".log."..i - local iow, err = f:iowatcher { - key = key, - kz_flags = arg.oneshot and { "nofollow" }, - handler = function (iow, r, err) - if not r then - if err then - io.stderr:write ("logstream kz error "..err.."\n") - end - return - end - logstream_print_line (iow, r) - end - } - iow.id = i - table.insert (l.watchers, iow) - end - return setmetatable (l, logstream) -end - -local function exit_message (t) - local s = "exited with " - s = s .. (t.exit_code and "exit code" or "signal") .. " %d" - return s:format (t.exit_code or t.exit_sig) -end - -local function task_status (lwj, taskid) - if not tonumber (taskid) then return nil end - local t = lwj[taskid] - if not t.exit_status then - return 0, "running" - end - local x = t.exit_code or (t.exit_sig + 128) - return x, exit_message (t) -end - -local function exit_status_aggregate (arg) - local flux = require 'flux' - local f = arg.flux - local lwj = arg.kvsdir - local msgs = {} - - local exit_status = lwj.exit_status - if not exit_status then return nil, nil end - - local s, max, core = flux.exitstatus (exit_status.max) - if s == "killed" then max = max + 128 end - - for ids,v in pairs (exit_status.entries) do - local msg, code, core = flux.exitstatus (v) - s = "exited with " - s = s .. (msg == "exited" and "exit code " or "signal ") .. code - msgs [s] = hostlist.new (ids) - end - - return max, msgs -end - - ---- Return highest exit code from all tasks and task exit message list. --- Summarize job exit status for arg.jobid by returning max task exit code, --- along with a list of task exit messages to be optionally emitted to stdout. --- @param arg.jobid job identifier --- @param arg.flux (optional) flux handle --- @return exit_cde, msg_list -function wreck.status (arg) - local flux = require 'flux' - local f = arg.flux - local jobid = arg.jobid - if not jobid then return nil, "required arg jobid" end - - if not f then f = require 'flux'.new() end - local lwj = f:kvsdir (kvs_path (f, jobid)) - - local max = 0 - local msgs = {} - - if lwj.exit_status then - return exit_status_aggregate { flux = f, kvsdir = lwj } - end - - for taskid in lwj:keys () do - local x, s = task_status (lwj, taskid) - if x then - if x > max then max = x end - if not msgs[s] then - msgs[s] = hostlist.new (taskid) - else - msgs[s]:concat (taskid) - end - end - end - return max, msgs -end - -function wreck.id_to_path (arg) - local flux = require 'flux' - local f = arg.flux - local id = arg.jobid - if not id then return nil, "required arg jobid" end - if not f then f, err = require 'flux'.new () end - if not f then return nil, err end - return kvs_path (f, id) -end - -local function kvsdir_reverse_keys (dir) - local result = {} - for k in dir:keys () do - if tonumber (k) then - table.insert (result, k) - end - end - table.sort (result, function (a,b) return tonumber(b) < tonumber(a) end) - return result -end - -local function reverse (t) - local len = #t - local r = {} - for i = len, 1, -1 do - table.insert (r, t[i]) - end - return r -end - --- append array t2 to t1 -local function append (t1, t2) - for _,v in ipairs (t2) do table.insert (t1, v) end - return t1 -end - -function wreck.jobids_to_kvspath (arg) - local f = arg.flux - local ids = arg.jobids - if not ids then return nil, 'missing required arg jobids' end - if not f then f, err = require 'flux'.new () end - if not f then return nil, err end - return kvs_path_multi (f, ids) -end - -local function include_state (conf, state) - -- Include all states if no state conf is supplied - if not conf then return true end - - -- Otherwise, if there is a conf.include hash then *only* include - -- states that appear in this hash: - if conf.include then return conf.include[state] end - - -- Otherwise, exclude this state if it is in the conf.exclude hash: - if conf.exclude then return not conf.exclude[state] end - - -- No conf.include, no conf.exclude, I guess it is included: - return true -end - --- Convert keys of table t to a comma-separated list of strings -local function to_string_list (t) - if not t then return nil end - local r = {} - for k,v in pairs (t) do - if v then table.insert (r, k) end - end - return table.concat (r, ",") -end - --- Return a list and hash of active jobs by kvs path -local function joblist_active (arg) - local f = arg.flux - local conf = arg.states - local r, err = f:rpc ("job.list", - { max = arg.max, - include = to_string_list (conf and conf.include), - exclude = to_string_list (conf and conf.exclude) - }) - if not r then return nil, nil, err end - local active = {} - local results = {} - for _,job in ipairs (r.jobs) do - local path = job.kvs_path - kvs_path_insert (job.jobid, path) - active[path] = true - table.insert (results, path) - end - return results, active -end - -local function joblist_kvs (arg, exclude) - local flux = require 'flux' - local f = arg.flux - if not f then f, err = flux.new () end - if not f then return nil, err end - - local function visit (d, r) - local results = r or {} - if not d then return end - local dirs = kvsdir_reverse_keys (d) - for _,k in pairs (dirs) do - local path = tostring (d) .. "." .. k - local dir = f:kvsdir (path) - if dir and (not exclude or not exclude[path]) then - if dir.state then - -- This is a lwj dir, add to results table: - if include_state (arg.states, dir.state) then - table.insert (results, path) - end - else - -- recurse to find lwj dirs lower in the directory tree - visit (dir, results) - end - if arg.max and #results >= arg.max then return results end - end - end - return results - end - - local dir, err = f:kvsdir ("lwj") - if not dir then - if err:match ("No such") then err = "No job information in KVS" end - return nil, err - end - - return reverse (visit (dir)) -end - -function wreck.joblist (arg) - local results, active = {}, {} - if not arg.kvs_only then - results, active = joblist_active (arg) - if results and arg.max then - arg.max = arg.max - #results - end - if arg.active_only or arg.max == 0 then return results end - end - - -- Append a list of jobs from the kvs to the active jobs - local r, err = joblist_kvs (arg, active) - if not r then return nil, err end - - return append (results or {}, r) -end - -local function shortprog () - local prog = string.match (arg[0], "([^/]+)$") - return prog:match ("flux%-(.+)$") -end - -function wreck.new (arg) - if not arg then arg = {} end - local w = setmetatable ({ extra_options = {}, - kvsdirs = {}, - lwj_paths = {}}, wreck) - w.prog = arg.prog or shortprog () - w.flux = arg.flux - return w -end - -return wreck - --- vi: ts=4 sw=4 expandtab diff --git a/src/bindings/lua/wreck/io.lua b/src/bindings/lua/wreck/io.lua deleted file mode 100644 index f22298a85309..000000000000 --- a/src/bindings/lua/wreck/io.lua +++ /dev/null @@ -1,369 +0,0 @@ -------------------------------------------------------------- --- Copyright 2014 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - --- --- Task I/O library for WRECK --- - - --- --- ostream - simple output stream with reference count. --- -local ostream = {} -ostream.__index = ostream -function ostream.create (filename) - local s = { count = 0, filename = filename, flag = "w" } - setmetatable (s, ostream) - return s -end - -function ostream:closed () - return self.count == 0 -end - -function ostream:open () - if not self.fp then - if self.filename == "stdout" or self.filename == "stderr" then - self.fp = io[self.filename] - else - self.fp, err = io.open (self.filename, self.flag) - self.fp:setvbuf 'line' - end - if not self.fp then return nil, self.filename..": "..err end - end - self.count = self.count + 1 - return true -end - -function ostream:close () - self.count = self.count - 1 - if self.count == 0 then - self.fp:close () - self.fp = nil - end -end - -function ostream:write (data) - if not self.fp then return nil, "File not open" end - local rc, err = self.fp:write (data) - if not rc then print (err) end -end - - -local kvsfile = {} -kvsfile.__index = kvsfile - -function kvsfile.create (arg) - local s = { count = 0, key = arg.key, flux = arg.flux, data = "" } - setmetatable (s, kvsfile) - return s -end - -function kvsfile:open () - self.count = self.count + 1 - return true -end - -function kvsfile:close () - self.count = self.count - 1 - if self.count == 0 then - local rc, err = self.flux:kvs_put (self.key, self.data) - if not rc then io.stderr:write (err) end - self.flux:kvs_commit () - end -end - -function kvsfile:closed () - return self.count == 0 -end - -function kvsfile:write (data) - self.data = self.data .. data -end - --- --- wreck job io tracking object. --- Direct all task io to a named set of ostream objects --- -local ioplex = {} -ioplex.__index = ioplex - - ---- Create a wreck i/o multiplexer --- ioplex objects are an abstraction of wreck reactor-driven I/O --- with many-to-one or many-to-many output redirection. Named --- output stream objects are registered with an ioplex object, --- and then task "stdout" and "stderr" streams are redirected --- onto these output streams. Output streams have reference counting --- and are only closed when all references have been closed. --- When all outputs are closed ioplex:complete() will return true. ---@param arg a Table of named keys for arguments ---@return ioplex object --- -function ioplex.create (arg) - if not arg.flux or not arg.jobid then - return nil, "Required handle or jobid arg missing!" - end - - local io = { - flux = arg.flux, - id = arg.jobid, - labelio = arg.labelio, - kvspath = arg.kvspath, - on_completion = arg.on_completion, - log_err = arg.log_err, - nofollow = arg.nofollow, - nokz = arg.nokz, - removed = {}, - output = {}, - files = {} - } - if not io.kvspath then - local wreck = require 'wreck' - local r, err = wreck.id_to_path { flux = io.flux, jobid = io.id } - if not r then error (err) end - io.kvspath = r - end - if io.nokz then - if not arg.ioservices then return nil, "required arg ioservices" end - io.ioservices = {} - for s,v in pairs (arg.ioservices) do - if v and v.rank == io.flux.rank then - io.ioservices[s] = v.name - end - end - end - setmetatable (io, ioplex) - return io -end - -local function eprintf (...) - io.stderr:write (string.format (...).."\n") -end - -function ioplex:enable_debug (logger) - self.logger = logger and logger or eprintf -end - -function ioplex:log (fmt, ...) - if self.logger then - self.logger (fmt, ...) - end -end - -function ioplex:err (fmt, ...) - if self.log_err then - self.log_err (fmt, ...) - else - io.stderr:write (string.format (fmt, ...)) - end -end - - -local function ioplex_set_stream (self, taskid, name, f) - if not self.output[taskid] then - self.output[taskid] ={} - end - self.output[taskid][name] = f -end - -local function ioplex_open_path (self, path) - local f, err - local key = path:match ("^kvs://(.+)$") - if not key then - self:log ("creating path %s", path) - return ostream.create (path) - else - self:log ("opening kvs key %s", key) - return kvsfile.create { flux = self.flux, key = key } - end -end - -local function ioplex_create_stream (self, path) - local files = self.files - if not files[path] then - local f, err = ioplex_open_path (self, path) - if not f then return nil, err end - files[path] = f - end - return files[path] -end - -function ioplex:close_output (of) - of:close () - if of:closed () then - self:log ("closed path %s", of.filename) - end - if self:complete() and self.on_completion then - self.on_completion() - end -end - -function ioplex:output_handler (arg) - local taskid = arg.taskid or 0 - local stream = arg.stream or "stdout" - local data = arg.data - local of = arg.output or self.output[taskid][stream] - - if not data then - self:close_output (of) - return - end - if self.labelio then - of:write (taskid..": ") - end - of:write (data) -end - -function ioplex:register_services () - local f = self.flux - if not self.ioservices then return nil, "IO Service not started yet!" end - - for _,service in pairs (self.ioservices) do - local rc, err = f:rpc ("service.add", { service = service }) - if not rc then - return nil, err - end - end - return true -end - -function ioplex:msg_handler (stream, zmsg) - local data = nil - local msg = zmsg.data - if not msg then return end - - if not msg.eof then - data = msg.data - end - self:output_handler { taskid = msg.taskid, - data = data, - stream = stream } -end - -function ioplex:ioservice_start () - -- I/O services for stdout/err for this job: - for stream,service in pairs (self.ioservices) do - local mh, err = self.flux:msghandler { - pattern = service..".write", - handler = function (f, zmsg, mh) - self:msg_handler (stream, zmsg) - end - } - if not mh then self:err ("ioservice_start: %s", err) end - end -end - -local function ioplex_taskid_start (self, flux, taskid, stream) - local of = self.output[taskid][stream] - if not of then return nil, "No stream "..stream.." for task " .. taskid end - - local flags = {} - if self.nofollow then table.insert (flags, "nofollow") end - - local f = flux - local key = string.format ("%s.%d.%s", self.kvspath, taskid, stream) - local iow, err = f:iowatcher { - key = key, - kz_flags = flags, - handler = function (iow, data, err) - self:output_handler { taskid = taskid, - stream = stream, - output = of, - data = data } - end - } - if not iow then - self:log ("ignoring %s: %s", key, err) - -- remove reference count - of:close () - end -end - -function ioplex:closeall (stream) - for taskid, t in pairs (self.output) do - if t[stream] then t[stream]:close () end - end -end - -function ioplex:close_unwatched_streams () - for _,stream in pairs { "stdout", "stderr" } do - if not self.ioservices [stream] then self:closeall (stream) end - end -end - ---- "start" all ioplex iowatchers on reactor. --- @param h flux handle of reactor to use (optional) -function ioplex:start (h) - local flux = h or self.flux - if not flux then return nil, "Error: no flux handle!" end - - if self.nokz then - self:ioservice_start () - local rc, err = self:register_services () - if not rc then - self:err ("Unable to start I/O service: %s", err) - end - self:close_unwatched_streams () - return - end - - for taskid, t in pairs (self.output) do - for stream, f in pairs (t) do - ioplex_taskid_start (self, flux, taskid, stream) - end - end - self.started = true -end - ---- redirect a stream from a task to named stream "path". --- On first reference to `path` (not including special paths "stdout" --- and "stderr") the referenced file will be opened with `io.open(..., "w")` --- --- @param taskid task identifier --- @param stream name of stream to redirect (e.g. "stdout") --- @param path path of file to redirect to (or "stderr" "stdout") --- @return true on success, nil with error on failure --- -function ioplex:redirect (taskid, stream, path) - local f, err = ioplex_create_stream (self, path) - if not f then return nil, err end - - ioplex_set_stream (self, taskid, stream, f) - return f:open() -end - ---- duplicate a stream from a task onto existing stream for this task. --- Reference count is incremented for `dst` --- @param taskid task identifier --- @param src the source stream to duplicate (e.g. "stderr") --- @param dst the destination stream to copyto (e.g. "stdout") --- -function ioplex:dup (taskid, src, dst) - local f = self.output[taskid][dst] - if not f then - return nil, 'No such output stream: '..dst - end - self.output[taskid][src] = f - return f:open() -end - ---- Check for completion of I/O on an ioplex object --- @return true if IO is complete, false if not -function ioplex:complete () - for _,f in pairs (self.files) do - if not f:closed() then return false end - end - return true -end - -return ioplex - --- vi: ts=4 sw=4 expandtab diff --git a/src/bindings/python/flux/Makefile.am b/src/bindings/python/flux/Makefile.am index 03148eb206bc..10c5c071ec20 100644 --- a/src/bindings/python/flux/Makefile.am +++ b/src/bindings/python/flux/Makefile.am @@ -5,7 +5,6 @@ fluxpy_PYTHON=\ rpc.py\ message.py\ constants.py\ - jsc.py\ kz.py\ job.py \ mrpc.py \ diff --git a/src/bindings/python/flux/__init__.py b/src/bindings/python/flux/__init__.py index 0dde744db3d0..4e652dd25313 100644 --- a/src/bindings/python/flux/__init__.py +++ b/src/bindings/python/flux/__init__.py @@ -19,4 +19,4 @@ def Flux(*args, **kwargs): return flux.core.handle.Flux(*args, **kwargs) -__all__ = ["core", "kvs", "jsc", "rpc", "mrpc", "constants", "Flux"] +__all__ = ["core", "kvs", "rpc", "mrpc", "constants", "Flux"] diff --git a/src/bindings/python/flux/jsc.py b/src/bindings/python/flux/jsc.py deleted file mode 100644 index f3e5aec3871d..000000000000 --- a/src/bindings/python/flux/jsc.py +++ /dev/null @@ -1,89 +0,0 @@ -############################################################### -# Copyright 2014 Lawrence Livermore National Security, LLC -# (c.f. AUTHORS, NOTICE.LLNS, COPYING) -# -# This file is part of the Flux resource manager framework. -# For details, see https://github.com/flux-framework. -# -# SPDX-License-Identifier: LGPL-3.0 -############################################################### - -import json -import re -import sys - -import six - -from flux.wrapper import Wrapper -from _flux._core import ffi, lib - -MOD = sys.modules[__name__] -# Inject enum/define names matching ^JSC_[A-Z_]+$ into module -PATTERN = re.compile("^JSC_[A-Z_]+") -for k in dir(lib): - if PATTERN.match(k): - v = ffi.string(getattr(lib, k)).decode("ascii") - setattr(MOD, k, v) - - -class JSCWrapper(Wrapper): - """ - Generic JSC wrapper - """ - - def __init__(self): - """Set up the wrapper interface for functions prefixed with jsc_""" - super(JSCWrapper, self).__init__(ffi, lib, prefixes=["jsc_"]) - - -RAW = JSCWrapper() - - -def query_jcb(flux_handle, jobid, key): - jcb_str = ffi.new("char *[1]") - RAW.query_jcb(flux_handle, jobid, key, jcb_str) - if jcb_str[0] == ffi.NULL: - return None - return json.loads(ffi.string(jcb_str[0]).decode("utf-8")) - - -def update_jcb(flux_handle, jobid, key, jcb): - # TODO: validate the key is one of the constants in jstatctl.h - if isinstance(jcb, six.text_type): - jcb = jcb.encode("utf-8") - elif not isinstance(jcb, six.binary_type): - jcb = json.dumps(jcb).encode("utf-8") - return RAW.jsc_update_jcb(flux_handle, jobid, key, jcb) - - -@ffi.callback("jsc_handler_f") -def jsc_notify_wrapper(jcb, arg, errnum): - if jcb != ffi.NULL: - jcb = ffi.string(jcb).decode("utf-8") - callback, real_arg = ffi.from_handle(arg) - ret = callback(jcb, real_arg, errnum) - return ret if ret is not None else 0 - - -HANDLES = [] - - -def notify_status(flux_handle, fun, arg): - warg = (fun, arg) - whandle = ffi.new_handle(warg) - HANDLES.append(whandle) - return RAW.notify_status(flux_handle, jsc_notify_wrapper, whandle) - - -def job_num2state(job_state): - ret = RAW.job_num2state(job_state) - if ret == ffi.NULL: - return None - return ret.decode("ascii") - - -def job_state2num(job_state): - if isinstance(job_state, six.text_type): - # jsc doesn't use utf-8 internally, catch erroneous unicode here - job_state.encode("ascii") - return RAW.job_state2num(job_state) diff --git a/src/broker/broker.c b/src/broker/broker.c index dff1a0d52957..528e477f0345 100644 --- a/src/broker/broker.c +++ b/src/broker/broker.c @@ -686,8 +686,6 @@ static struct attrmap attrmap[] = { { "FLUX_PMI_LIBRARY_PATH", "conf.pmi_library_path", 1 }, { "FLUX_RC1_PATH", "broker.rc1_path", 1 }, { "FLUX_RC3_PATH", "broker.rc3_path", 1 }, - { "FLUX_WRECK_LUA_PATTERN", "wrexec.lua_pattern", 1 }, - { "FLUX_WREXECD_PATH", "wrexec.wrexecd_path", 1 }, { "FLUX_SEC_DIRECTORY", "security.keydir", 1 }, { "FLUX_URI", "parent-uri", 0 }, @@ -699,8 +697,6 @@ static void init_attrs_from_environment (attr_t *attrs) struct attrmap *m; const char *val; int flags = 0; // XXX possibly these should be immutable? - // however they weren't before and wreck test depends - // on changing wrexec.lua_pattern for (m = &attrmap[0]; m->env != NULL; m++) { val = getenv (m->env); diff --git a/src/cmd/.gitignore b/src/cmd/.gitignore index f58fa42ec295..bf78335bd263 100644 --- a/src/cmd/.gitignore +++ b/src/cmd/.gitignore @@ -3,7 +3,6 @@ /flux-comms-stats /flux-event /flux-jobspec-validate -/flux-jstat /flux-keygen /flux-kvs /flux-logger diff --git a/src/cmd/Makefile.am b/src/cmd/Makefile.am index ad6d9fef5ab5..35907aaef87d 100644 --- a/src/cmd/Makefile.am +++ b/src/cmd/Makefile.am @@ -54,9 +54,6 @@ nodist_flux_SOURCES = \ # dist_fluxcmd_SCRIPTS = \ - flux-submit \ - flux-wreckrun \ - flux-wreck \ flux-cron \ flux-aggregate \ flux-hostlist @@ -70,7 +67,6 @@ fluxcmd_PROGRAMS = \ flux-comms \ flux-kvs \ flux-start \ - flux-jstat \ flux-job \ flux-exec diff --git a/src/cmd/flux-hostlist b/src/cmd/flux-hostlist index 3ba18fc5b0fe..f568c577b486 100755 --- a/src/cmd/flux-hostlist +++ b/src/cmd/flux-hostlist @@ -11,12 +11,11 @@ ------------------------------------------------------------- -- --- flux-hostlist: print list of hosts in hostfile format for instance or job +-- flux-hostlist: print list of hosts in hostfile format for instance -- -- local getopt = require 'flux.alt_getopt'.get_opts local hostlist = require 'flux.hostlist' -local wreck = require 'wreck' local prog = "flux-hostlist" local hostkey = "resource.hosts" @@ -71,61 +70,6 @@ local function instance_host_table (arg) return hosts end --- Get R_lite at arg.kvspath and return comma-separated list of hostnames -local function R_lite_to_hosts (arg) - local f = arg.flux - local dir = arg.kvspath - local hosts = {} - local allhosts - - local R, err = f:kvs_get (dir..".R_lite") - if not R then die ("Failed to get R_lite at %s: %s\n", dir, err) end - - for _,entry in ipairs (R) do - if arg.ranks then - table.insert (hosts, entry.rank) - elseif entry.node then - table.insert (hosts, entry.node) - else - local allhosts = instance_host_table { flux = f } - table.insert (hosts, allhosts [entry.rank+1]) - end - end - return table.concat (hosts, ",") -end - --- Return a list of hosts for jobids given in arg.args list. -local function jobs_host_table (arg) - local f = arg.flux - local args = arg.args - local ranks = arg.ranks - local function toint (t) - local r = {} - for _,v in ipairs (t) do table.insert (r, tonumber (v)) end - return r - end - local function bracketify (t) - local r = {} - for _,v in ipairs (t) do table.insert (r, "["..v.."]") end - return r - end - local hl, err = hostlist.union (unpack (bracketify (args))) - if not hl then die ("hostlist.union: %s\n", err) end - - local kvspaths = wreck.jobids_to_kvspath { flux = f, - jobids = toint (hl:expand()) - } - if not kvspaths then die ("wreck.jobids_to_kvspath failed") end - - local hosts = hostlist.new () - - for _,p in ipairs (kvspaths) do - hosts:concat (R_lite_to_hosts { flux = f, kvspath = p, ranks = ranks }) - end - return hosts:expand() -end - - local function hosts_from_stream (fp) local hosts = {} for line in fp:lines() do @@ -164,8 +108,8 @@ local function set_hostlist (arg) end local function usage (code) - printf ("Usage: %s [OPTIONS] [JOBIDS]...\n", prog) - printf ("Get/set a list of hosts for the current session or job(s)\n") + printf ("Usage: %s [OPTIONS]...\n", prog) + printf ("Get/set a list of hosts for the current session\n") printf ("Options:\n") printf (" -h, --help Display this message.\n") printf (" -r, --ranks Output rank ids instead of hostnames.\n") @@ -199,14 +143,8 @@ end for i = 1,optind-1 do table.remove (arg, 1) end --- Otherwise, read either the instance hosts table or job --- specific hosts lists and emit one host per line to stdout: -local hosts = {} -if #arg == 0 then - hosts = instance_host_table { flux = f, ranks = opts.r } -else - hosts = jobs_host_table { flux = f, ranks = opts.r, args = arg } -end +-- read the instance hosts table +local hosts = instance_host_table { flux = f, ranks = opts.r } -- Print result: if opts.c then diff --git a/src/cmd/flux-jstat.c b/src/cmd/flux-jstat.c deleted file mode 100644 index 35dc555458a4..000000000000 --- a/src/cmd/flux-jstat.c +++ /dev/null @@ -1,270 +0,0 @@ -/************************************************************\ - * Copyright 2014 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "src/common/libutil/log.h" -#include "src/common/libutil/xzmalloc.h" - - -/****************************************************************************** - * * - * Internal types, macros and static variables * - * * - ******************************************************************************/ -typedef struct { - flux_t *h; - FILE *op; -} jstatctx_t; - -/* Note: Should only be used for signal handling */ -static flux_t *sig_flux_h; - -#define OPTIONS "o:h" -static const struct option longopts[] = { - {"help", no_argument, 0, 'h'}, - {"testout", required_argument, 0, 'o'}, - { 0, 0, 0, 0 }, -}; - - -/****************************************************************************** - * * - * Utilities * - * * - ******************************************************************************/ -static void usage (int code) -{ - fprintf (stderr, -"Usage: flux-jstat notify\n" -" flux-jstat query jobid \n" -" flux-jstat update jobid \n" -); - exit (code); -} - -static void freectx (void *arg) -{ - jstatctx_t *ctx = arg; - if (ctx->op) - fclose (ctx->op); -} - -static jstatctx_t *getctx (flux_t *h) -{ - jstatctx_t *ctx = (jstatctx_t *)flux_aux_get (h, "jstat"); - if (!ctx) { - ctx = xzmalloc (sizeof (*ctx)); - ctx->h = h; - ctx->op = NULL; - flux_aux_set (h, "jstat", ctx, freectx); - } - return ctx; -} - -static void sig_handler (int s) -{ - if (s == SIGINT) { - fprintf (stdout, "Exit on INT"); - exit (0); - } -} - -static FILE *open_test_outfile (const char *fn) -{ - FILE *fp; - if (!fn) - fp = NULL; - else if ( !(fp = fopen (fn, "w"))) - fprintf (stderr, "Failed to open %s\n", fn); - return fp; -} - -/****************************************************************************** - * * - * Async notification callback * - * * - ******************************************************************************/ - -static int job_status_cb (const char *jcbstr, void *arg, int errnum) -{ - int os; - int ns; - jstatctx_t *ctx = NULL; - flux_t *h = (flux_t *)arg; - json_t *jcb = NULL; - - ctx = getctx (h); - if (errnum > 0) { - flux_log (ctx->h, LOG_ERR, "job_status_cb: errnum passed in"); - return -1; - } - - if (!(jcb = json_loads (jcbstr, 0, NULL)) - || json_unpack (jcb, "{s:{s:i s:i}}", JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, &os, - JSC_STATE_PAIR_NSTATE, &ns) < 0) { - flux_log (ctx->h, LOG_ERR, "job_status_cb: error parsing JSON string"); - json_decref (jcb); - return -1; - } - - fprintf (ctx->op, "%s->%s\n", jsc_job_num2state (os), - jsc_job_num2state (ns)); - fflush (ctx->op); - json_decref (jcb); - return 0; -} - - -/****************************************************************************** - * * - * Top-level Handlers for each of notify, query and update commands * - * * - ******************************************************************************/ - -static int handle_notify_req (flux_t *h, const char *ofn) -{ - jstatctx_t *ctx = NULL; - - sig_flux_h = h; - if (signal (SIGINT, sig_handler) == SIG_ERR) - return -1; - - ctx = getctx (h); - ctx->op = (ofn)? open_test_outfile (ofn) : stdout; - - if (jsc_notify_status (h, job_status_cb, (void *)h) != 0) { - flux_log (h, LOG_ERR, "failed to reg a job status change CB"); - return -1; - } - if (flux_reactor_run (flux_get_reactor (h), 0) < 0) - flux_log (h, LOG_ERR, "error in flux_reactor_run"); - - return 0; -} - -static int handle_query_req (flux_t *h, int64_t j, const char *k, const char *n) -{ - jstatctx_t *ctx = getctx (h); - json_t *jcb = NULL; - char *jcbstr = NULL; - char *jcbstr_pretty = NULL; - - ctx->op = n ? open_test_outfile (n) : stdout; - if (jsc_query_jcb (h, j, k, &jcbstr) != 0) { - flux_log (h, LOG_ERR, "jsc_query_jcb reported an error"); - return -1; - } - if (!(jcb = json_loads (jcbstr, 0, NULL))) { - flux_log (h, LOG_ERR, "error loading jcbstr"); - goto done; - } - if (!(jcbstr_pretty = json_dumps (jcb, JSON_INDENT(2)))) { - flux_log (h, LOG_ERR, "error dumping jcbstr"); - goto done; - } - - fprintf (ctx->op, "Job Control Block: attribute %s for job %"PRIi64"\n", k, j); - fprintf (ctx->op, "%s\n", jcbstr_pretty); -done: - free (jcbstr_pretty); - json_decref (jcb); - free (jcbstr); - return 0; -} - -static int handle_update_req (flux_t *h, int64_t j, const char *k, - const char *jcbstr, const char *n) -{ - jstatctx_t *ctx = NULL; - ctx = getctx (h); - ctx->op = n? open_test_outfile (n) : stdout; - - return (jsc_update_jcb (ctx->h, j, k, jcbstr)); -} - - -/****************************************************************************** - * * - * Main entry point * - * * - ******************************************************************************/ - -int main (int argc, char *argv[]) -{ - flux_t *h; - int ch = 0; - int rc = 0; - char *cmd = NULL; - const char *j = NULL; - const char *ofn = NULL; - const char *attr = NULL; - const char *jcbstr = NULL; - - log_init ("flux-jstat"); - while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) { - switch (ch) { - case 'h': /* --help */ - usage (0); - break; - case 'o': /* --testout */ - ofn = xasprintf ("%s", optarg); - break; - default: - usage (1); - break; - } - } - if (optind == argc) - usage (1); - - if (!(h = flux_open (NULL, 0))) - log_err_exit ("flux_open"); - - flux_log_set_appname (h, "jstat"); - cmd = argv[optind++]; - - if (!strcmp ("notify", cmd)) - rc = handle_notify_req (h, (const char *)ofn); - else if (!strcmp ("query", cmd) && optind == argc - 2) { - j = (const char *)(*(argv+optind)); - attr = (const char *)(*(argv+optind+1)); - rc = handle_query_req (h, strtol (j, NULL, 10), attr, ofn); - } - else if (!strcmp ("update", cmd) && optind == argc - 3) { - j = (const char *)(*(argv+optind)); - attr = (const char *)(*(argv+optind+1)); - jcbstr = (const char *)(*(argv+optind+2)); - rc = handle_update_req (h, strtol (j, NULL, 10), attr, jcbstr, ofn); - } - else - usage (1); - - flux_close (h); - log_fini (); - - return (!rc)? 0: 42; -} - -/* - * vi:tabstop=4 shiftwidth=4 expandtab - */ diff --git a/src/cmd/flux-submit b/src/cmd/flux-submit deleted file mode 100755 index 896024a5c1db..000000000000 --- a/src/cmd/flux-submit +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env lua - -------------------------------------------------------------- --- Copyright 2015 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - --- --- flux-submit: submit jobs using job.submit interface --- -local wreck = require 'wreck' .new () -if not wreck:parse_cmdline (arg) then - wreck:die ("Failed to process cmdline args\n") -end - --- If 'nokz' is used without --output, then store stdout and err in --- well-known kvs keys by default: -if wreck.job_options.nokz and not wreck:getopt('O') then - wreck.opts.O = "kvs://files.stdout" - if not wreck:getopt ('E') then wreck.opts.E = "kvs://files.stderr" end - - -- Regenerate wreck.output output file configuration: - wreck.output = wreck:output_file_config () -end - -local id, err = wreck:submit () -if not id then wreck:die ("%s\n", err) end -wreck:say ("Submitted jobid %d\n", id) -os.exit (0) --- vi: ts=4 sw=4 expandtab diff --git a/src/cmd/flux-wreck b/src/cmd/flux-wreck deleted file mode 100755 index f1ae9c05406b..000000000000 --- a/src/cmd/flux-wreck +++ /dev/null @@ -1,909 +0,0 @@ -#!/usr/bin/env lua - -------------------------------------------------------------- --- Copyright 2014 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - --- --- flux-wreck: simple wreck job utility --- -local f, err = require 'flux' .new() -local wreck = require 'wreck' -local hostlist = require 'flux.hostlist' -local posix = require 'flux.posix' -local prog = require 'flux.Subcommander'.create { - usage = "COMMAND [OPTIONS]", - handler = function (self, arg) - self:log ("COMMAND required\n") - self:help (1) - end -} - -local function printf (...) - io.stdout:write (string.format (...)) -end - -local function check_jobid_arg (self, id) - if not id then - io.stderr:write ("Error: Job id required\n") - self:help (1) - elseif not tonumber (id) then - self:die ("Invalid Job id '%s'\n", id) - end - return id -end - - -local function kvs_path (id, fmt, ...) - local id = assert (tonumber (id)) - local p, err = assert (wreck.id_to_path { flux = f, jobid = id }) - if fmt then - p = p .. "." .. string.format (fmt, ...) - end - return p -end - -local function dump_nokz_output (id) - local files = kvs_path (id, "files") - local stdout = f:kvs_get (files .. ".stdout") - local stderr = f:kvs_get (files .. ".stderr") - if stdout then io.stdout:write (stdout) end - if stderr then io.stderr:write (stderr) end -end - -local function attach_job_complete (state, taskio) - if not state then return false end - if state == "complete" or state == "reaped" then - if not taskio then return true end - return taskio:complete() - end - return false -end - -prog:SubCommand { - name = "attach", - description = "Attach to running or completed job output", - usage = "[OPTIONS] JOBID", - options = { - { name = "status", char = "s", - usage = "Include status after all output" }, - { name = "label-io", char = "l", - usage = "Label lines of output with task id" }, - { name = "no-follow", char = "n", - usage = "Don't wait for EOF on all output streams before exiting" }, - }, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local state = f:kvs_get (kvs_path (id, "state")) - local options = f:kvs_get (kvs_path (id, "options")) - local taskio - - if options.nokz and self.opt.l then - self:die ("--label-io not possible for jobs with 'nokz' option") - end - - if not options.nokz then - local err - taskio, err = wreck.ioattach { - flux = f, - jobid = id, - labelio = self.opt.l, - nofollow = self.opt.n, - on_completion = function () - if self.opt.n or attach_job_complete (state, taskio) then - f:reactor_stop() - end - end - } - if not taskio then - self:die ("Failed to connect to job %d: %s\n", id, err) - end - end - - local log, err = wreck.logstream { flux = f, jobid = id } - if not log then - self:die ("Failed to open logstream to job %d: %s\n", id, err) - end - - if state ~= "complete" and state ~= "reaped" then - local kz, err = f:kz_open (kvs_path (id, "input.files.stdin"), "w") - if kz then - local iow, err = f:iowatcher { - fd = posix.fileno (io.input()), - handler = function (iow, r) - if r.data then kz:write (r.data) end - if r.eof then kz:close () end - end - } - if not iow then self:die ("error opening stdin: %s\n", err) end - end - -- Watch state - local kw, err = f:kvswatcher { - key = kvs_path (id, "state"), - handler = function (kw, result) - if result then state = result end - if attach_job_complete (result, taskio) then - f:reactor_stop () - end - end - } - end - - if not attach_job_complete (state, taskio) then f:reactor () end - - if options.nokz then dump_nokz_output (id) end - log:dump() - if self.opt.s then - self.parent:run {"status", id} - end - os.exit (0) - end -} - -prog:SubCommand { - name = "dumplog", - description = "Dump error log stream for a wreck job", - usage = "[OPTIONS] JOBID", - options = nil, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local log, err = wreck.logstream { jobid = id, flux = f, oneshot = true } - if not log then - self:die ("Failed to open logstream to job %d: %s\n", id, err) - end - f:reactor() - end -} - -local function opt_sig (s) - if not s then return posix.SIGTERM end - if tonumber (s) then return tonumber (s) end - if not tonumber (posix[s]) then - prog:die ("Invalid signal '%s'\n", s) - end - return tonumber (posix [s]) -end - -prog:SubCommand { - name = "cancel", - description = "Cancel a pending job", - usage = "[-f|--force] JOBID", - options = { - { name = "force", char = "f", - usage = "Force cancel even if scheduler is not loaded" - }, - }, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local resp, err = f:rpc ("sched.cancel", { jobid = tonumber (id) }) - if not resp then - if err == "Function not implemented" and not self.opt.f then - prog:die ("job cancel not supported when scheduler not loaded") - elseif self.opt.f or err == "Invalid argument" then - prog:log ("Sending SIGKILL to %d\n", id) - local rc, err = f:sendevent ({signal = 9}, "wreck.%d.kill", id) - if not rc then self:die ("signal: %s\n", err) end - else - prog:die ("Unable to cancel %d: %s\n", id, err) - end - end - end -} - -prog:SubCommand { - name = "exclude", - description = "Exclude a node from running future jobs", - usage = "NODENAME", - options = { - { name = "kill", char = "k", - usage = "kill the jobs to which the node is allocated" - } - }, - handler = function (self, arg) - local function toboolean(X) - return not not X - end - - local nodename = arg[1] - if not nodename then - self:die ("Required argument NODENAME missing.") - end - local resp, err = f:rpc ("sched.exclude", { node = nodename, kill = toboolean (self.opt.k) }) - - if not resp then - if err == "Function not implemented" then - prog:die ("Node exclusion is not supported when scheduler not loaded") - else - prog:die ("Unable to exclude node %s: %s\n", nodename, err) - end - end - end -} - -prog:SubCommand { - name = "include", - description = "Include a node for scheduling", - usage = "NODENAME", - handler = function (self, arg) - local nodename = arg[1] - if not nodename then - self:die ("Required argument NODENAME missing.") - end - local resp, err = f:rpc ("sched.include", { node = nodename }) - if not resp then - if err == "Function not implemented" then - prog:die ("Node inclusion is not supported when scheduler not loaded") - else - prog:die ("Unable to include node %s: %s\n", nodename, err) - end - end - end -} - - -prog:SubCommand { - name = "kill", - description = "Kill a running job", - usage = "[OPTIONS] JOBID", - options = { - { name = "signal", char = "s", arg = "N", - usage = "Specify signal number N to be sent." } - }, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local lwj, err = f:kvsdir (kvs_path (id)) - if not lwj then self:die ("Job %d: %s\n", id, err) end - if lwj.state ~= "running" then - io.stderr:write ("Job "..id..": "..lwj.state.."\n") - os.exit (1) - end - local sig = opt_sig (self.opt.s) - local rc, err = f:sendevent ({signal = sig}, "wreck.%d.kill", id) - if not rc then self:die ("signal: %s\n", err) end - end -} - -prog:SubCommand { - name = "status", - description = "Return status of jobs", - usage = "JOBID", - handler = function (self, arg) - local jobid = check_jobid_arg (self, arg[1]) - local lwj, err = f:kvsdir (kvs_path (jobid)) - if not lwj then self:die ("Job %d: %s\n", jobid, err) end - print ("Job "..jobid.." status: "..lwj.state) - local code, msglist = wreck.status { flux = f, jobid = jobid } - if not code then self:die (msglist) end - for msg, hl in pairs (msglist) do - local s = string.format ("task%s: %s", tostring (hl:sort()), msg) - print (s) - end - os.exit (code) - end -} - - -local LWJ = {} -function LWJ.open (f, id, kvspath) - if not f or not id then - return nil, "Required arg missing" - end - if not kvspath then - kvspath = kvs_path (id) - end - local lwj, err = f:kvsdir (kvspath) - if not lwj then - return nil, "Failed to get lwj info: "..err - end - return setmetatable ({id = id, lwj = lwj }, LWJ) -end - -local function gettimeofday_as_table () - local ts, us = require 'flux.posix'.gettimeofday () - if not ts then return nil, us end - if type (ts) == "table" then return ts end - return { sec = ts, usec = us } -end - -local function now () - local t, err = gettimeofday_as_table () - if not t then error ("gettimeofday: "..err) end - return t.sec + (t.usec / (1000 * 1000)) -end - -function LWJ:timediff (tstart, tend, talt) - local t0 = self.lwj[tstart.."-time"] - if not t0 then return 0 end - local t1 = self.lwj[tend.."-time"] - if not t1 then t1 = self.lwj[talt.."-time"] end - if not t1 then t1 = now () end - local s = t1 - t0 - return s > 0 and s or 0 -end - -function LWJ:exit_string () - local flux = require 'flux' - local state = self.state - local exit_status = self.lwj.exit_status - if exit_status then - local max = exit_status.max - if max then - local s, code, core = flux.exitstatus (max) - state = s - if s == "exited" and code > 0 then - state = "failed" - end - end - end - return state -end - -function LWJ:state_string () - if self.state == "complete" then - return self:exit_string () - end - return self.state -end - -function LWJ:ranks () - local hl = hostlist.new() - local R = self.lwj.R_lite - if not R then - local rank = self.lwj.rank - if not rank then return nil end - for i in rank:keys() do hl:concat (i) end - else - for _,entry in ipairs (R) do - hl:concat (entry.rank) - end - end - return hl:sort() -end - -LWJ.__index = function (self, key) - if key == "state" then - if not self._state then - self._state = self.lwj.state - end - return self._state - elseif key == "ntasks" then - return self.lwj.ntasks - elseif key == "nnodes" then - return self.lwj.nnodes - elseif key == "runtime" then - return self:timediff ("starting", "complete", "failed") - elseif key == "t0" then - return self:timediff ("create", "starting", "failed") - elseif key == "t1" then - return self:timediff ("starting", "running", "failed") - elseif key == "t2" then - return self:timediff ("running", "complete", "failed") - elseif key == "start" then - return os.date ("%FT%T", self.lwj["starting-time"]) - elseif key == "end" then - return os.date ("%FT%T", self.lwj["complete-time"]) - elseif key == "created" then - return os.date ("%FT%T", self.lwj["create-time"]) - elseif key == "command" then - return self.lwj.cmdline[1] - elseif key == "name" then - return self.lwj.name - end - return LWJ [key] -end - -local function seconds_to_string (s) - local f = string.format - if s > (60*60*24) then - return f ("%.03fd", s / (60*60*24)) - elseif s > (60*60) then - return f ("%.03fh", s / (60*60)) - elseif s > 60 then - return f ("%.03fm", s / 60) - end - return f ("%.03fs", s) -end - -local function bracketify (args) - local r = {} - for _,v in ipairs (args) do - table.insert (r, "[" .. v .. "]") - end - return (r) -end - -local function joblist_from_args (arg) - local self = arg.self - local args = arg.args - - local max = tonumber (self.opt.n) or 25 - if max <= 0 then max = math.huge end - if #args == 0 then - return wreck.joblist{ flux = f, - max = max, - states = arg.states, - active_only = arg.active_only, - kvs_only = arg.kvs_only } - end - -- otherwise use dirs on cmdline - local hl,err = hostlist.union (unpack (bracketify (args))) - if not hl then return nil, err end - return wreck.jobids_to_kvspath { flux = f, jobids = hl:expand() } -end - -prog:SubCommand { - name = "ls", - usage = "[OPTIONS] [JOBIDs]", - options = { - { name = "max", char = 'n', arg="COUNT", - usage = "Display at most COUNT jobs", - }, - { name = "exclude-complete", char = 'x', - usage = "Skip the listing of complete jobs in the kvs" - } - }, - description = "List jobs in kvs", - handler = function (self, arg) - local dirs,err = joblist_from_args { self = self, args = arg, - active_only = self.opt.x } - if not dirs then self:die (err) end - if #dirs == 0 then return end - local fmt = "%6s %6s %-9s %20s %12s %8s %-.13s\n"; - printf (fmt, "ID", "NTASKS", "STATE", "START", - "RUNTIME", "RANKS", "COMMAND/NAME") - for _,dir in pairs (dirs) do - local id = dir:match ("(%d+)$") - if tonumber (id) then - local j, err = LWJ.open (f, id, dir) - if j then - printf (fmt, id, j.ntasks, j:state_string(), j.start, - seconds_to_string (j.runtime), - tostring (j:ranks()), - j.name or j.command:match ("([^/]+)$")) - end - end - end - end -} - -prog:SubCommand { - name = "uri", - usage = "[OPTIONS] [JOBIDs]", - options = { - { name = "max", char = 'n', arg="COUNT", - usage = "Display at most COUNT jobs", - }, - { name = "bare", char = 'b', - usage = "Dispaly only the URI", - } - }, - description = "List FLUX_URI for jobs that are Flux instances", - handler = function (self, arg) - -- Only include running jobs: - local states = { include = { running = true } } - local dirs,err = joblist_from_args { self = self, - args = arg, - states = states, - active_only = true } - if not dirs then self:die (err) end - if #dirs == 0 then return end - local fmt = "%6s %6s %-9s %-40s %-.13s\n"; - if self.opt.b then - if #dirs > 1 then self:die ("--bare only works with one job") end - else - printf (fmt, "ID", "NTASKS", "STATE", "FLUX_URI", "COMMAND") - end - for _,dir in pairs (dirs) do - local id = dir:match ("(%d+)$") - if tonumber (id) then - local j, err = LWJ.open (f, id, dir) - if not j then self:die ("job%d: %s", id, err) end - local uri, err = f:kvs_get (kvs_path (id, "flux.remote_uri")) - if self.opt.b then - if err then self:die ("job%d: not a Flux instance", id) end - printf ("%s\n", uri) - else - if err then uri = "" end - printf (fmt, id, j.ntasks, j:state_string(), uri, - j.command:match ("([^/]+)$")) - end - end - end - end -} - -prog:SubCommand { - name = "timing", - usage = "[OPTIONS] [JOBIDs]...", - options = { - { name = "max", char = 'n', arg="COUNT", - usage = "Display at most COUNT jobs", - } - }, - description = "List timings of jobs in kvs", - handler = function (self, arg) - local dirs,err = joblist_from_args {self = self, args = arg} - if not dirs then self:die (err) end - if #dirs == 0 then return end - local fmt = "%6s %12s %12s %12s %12s %12s\n" - printf (fmt, "ID", "NTASKS", "STARTING", "RUNNING", "COMPLETE", "TOTAL") - for _,dir in pairs (dirs) do - local id = dir:match ("(%d+)$") - if tonumber (id) then - local j, err = LWJ.open (f, id, dir) - if not j then self:die ("job%d: %s", id, err) end - printf (fmt, id, - j.ntasks, - seconds_to_string (j.t0), - seconds_to_string (j.t1), - seconds_to_string (j.t2), - seconds_to_string (j.runtime)) - end - end - end -} - -prog:SubCommand { - name = "kvs-path", - usage = "JOBID", - description = "Display path in KVS to job ID", - handler = function (self, arg) - if not arg[1] then self:die ("Job ID argument required\n") end - print (kvs_path (arg[1])) - end -} - -prog:SubCommand { - name = "last-jobid", - description = "Display the last jobid in lwj id sequence", - usage = "[OPTIONS]", - options = { - { name = "kvs-path", char = 'p', - usage = "Return path to job in kvs instead of jobid", - } - }, - handler = function (self, arg) - local req = { - name = "lwj", - preincrement = 0, - postincrement = 0, - create = false - } - local resp, err = f:rpc ("seq.fetch", req) - if not resp then - self:die ("No last jobid found: %s", err) - end - local result = resp.value - if self.opt.p then - result = kvs_path (result) - end - print (result) - end -} - -prog:SubCommand { - name = "setenv", - usage = "[VAR=VALUE|all]", - description = "Export environment to all jobs", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local getenv = wreck.get_filtered_env - local env = f:kvs_get ("lwj.environ") or {} - for _,expr in ipairs (arg) do - if expr == "all" then - for k,v in pairs(getenv()) do - env[k] = v - end - else - local var,val = expr:match("(.*)=(.*)") - env[var] = val - end - end - f:kvs_put ("lwj.environ", env) - f:kvs_commit () - end -} - -prog:SubCommand { - name = "unsetenv", - usage = "VAR [VAR...]", - description = "Export environment to all jobs", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local env = f:kvs_get ("lwj.environ") or {} - for _,var in ipairs (arg) do - env[var] = nil - end - f:kvs_put ("lwj.environ", env) - f:kvs_commit () - end -} - -prog:SubCommand { - name = "getenv", - usage = "VAR [VAR...]", - description = "Export environment to all jobs", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local env = f:kvs_get ("lwj.environ") or {} - if #arg == 0 then - for k,v in pairs (env) do - print (k.."="..v) - end - end - for _,k in ipairs (arg) do - local v = env[k] or "" - print (k.."="..v) - end - end -} - -prog:SubCommand { - name = "setopt", - usage = "option[=value]", - description = "Set global WRECK job options", - handler = function (self, arg) - if #arg == 0 then - self:die ("Missing option!") - end - local f = assert (require 'flux'.new ()) - local options = f:kvs_get ("lwj.options") or {} - for _,s in pairs (arg) do - local opt,val = s:match("([^=]+)=?(.*)") - if not val or val == "" then - options[opt] = true - elseif val == "false" or val == "no" or val == 0 then - options[opt] = nil - else - options[opt] = tonumber(val) or val - end - end - f:kvs_put ("lwj.options", options) - f:kvs_commit () - end -} - -prog:SubCommand { - name = "getopt", - usage = "[option]", - description = "Print global WRECK options", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local options = f:kvs_get ("lwj.options") or {} - if #arg == 0 then - for k,v in pairs (options) do - print (k.."="..tostring (v)) - end - else - for _,k in pairs (arg) do - v = options[k] - if v then - print (k.."="..tostring (v)) - else - print (k.."=false") - end - end - end - end -} - --- return keys in dir as a table sorted by number -local function sorted_keys (dir) - local results = {} - for k in dir:keys () do - table.insert (results, k) - end - table.sort (results, function (a,b) return tonumber (a) < tonumber (b) end) - return results -end - -local function remove_if_empty (key, r) - local results = r or {} - local dir = f:kvsdir (key) - if not dir or dir.state then return false, results end - local remove = true - for k in dir:keys () do - remove, results = remove_if_empty (key .. "." .. k, results) - end - if remove then - f:kvs_unlink (key) - table.insert (results, key) - end - return remove, results -end - -prog:SubCommand { - name = "purge", - usage = "[OPTIONS]", - options = { - { name = "verbose", char = "v", - usage = "Increase verbosity." }, - { name = "target-size", char = 't', arg = "N", - usage = "Specify a target number of lwj entries to keep" - }, - { name = "remove", char = "R", - usage = "Remove all eligible lwj entries." }, - { name = "max", char = "m", arg = "N", - usage = "Maximum number of entries to target." }, - { name = "keep", char = "k", arg = "CODE", - usage = "Keep entries where CODE is true" }, - }, - description = "Purge old jobs entries from kvs", - handler = function (self, arg) - local tt = require 'flux.timer'. new() - local verbose = self.opt.v - - --- - -- Gather LWJ path list - -- Only include complete jobs - local states = { include = { complete = true, - failed = true - } - } - local dirs = wreck.joblist{ flux = f, - kvs_only = true, - states = states - } or {} - if verbose then - self:log ("%4.03fs: got lwj list (%d entries)\n", tt:get0(), #dirs) - end - - local total = #dirs - local target = tonumber (self.opt.t) - local max = tonumber (self.opt.m) or math.huge - local keep = function (id) return false end - if self.opt.k then - local err - keep, err = - loadstring ('local id = tonumber (...); return '..self.opt.k) - if not keep then - self:die ("Failed to compile --keep option: %s\n", err) - end - end - - -- Without --target-size option, just list current number of lwj entries - if not target then - self:log ("%d total job entries\n", total) - elseif total <= target then - -- If current total is less than target, then log and exit - self:log ("%d total jobs at or below target size of %d\n", - total, target) - else - -- Otherwise, we need to purge some entries: - -- - local n = math.min (total - target, max) - - -- Remove the first n ids using lwj-completed directory - local r = {} - (function() -- anonymous function used for early return - local completed = f:kvsdir ("lwj-complete") - for _,hb in ipairs (sorted_keys (completed)) do - local hb_unlink = true - local d = completed [hb] - for _,id in ipairs (sorted_keys (d)) do - -- Save name of this kvs path: - local complink = tostring (d).."."..id - if keep (id) then - hb_unlink = false - else - table.insert (r, id) - if self.opt.R then - local t, target = f:kvs_type (complink) - if t == "symlink" then - f:kvs_unlink (target) - end - f:kvs_unlink (complink) - end - n = n - 1 - if n == 0 then return end - end - end - -- - -- Finished iterating all entries in this hb subdir, - -- should be ok to remove the hb dir now: - if self.opt.R and hb_unlink then - f:kvs_unlink ("lwj-complete."..hb) - end - end - end)() - - -- gather ids to remove in hostlist for condensed output: - -- - local hl = require 'flux.hostlist' .new (table.concat (r, ",")):uniq () - local rmdirs = {} - if self.opt.R then - f:kvs_commit() - if verbose then - self:log ("%4.03fs: unlinked %d entries\n", tt:get0(), #hl) - end - _, rmdirs = remove_if_empty ("lwj") - f:kvs_commit () - elseif verbose then - self:log ("%4.03fs: finished walking %d entries in lwj-complete\n", - tt:get0(), #hl) - end - - local idstring = "" - if #hl > 0 then - idstring = ": "..tostring (hl):match ("%[?([^]]+)%]?") - end - self:log ("%s %d lwj entries%s\n", - self.opt.R and "removed" or "would remove", #hl, - idstring) - if #rmdirs > 0 then - self:log ("removed %d empty dirs under lwj.\n", #rmdirs) - if verbose then - self:log ("removed: %s\n", table.concat (rmdirs, ", ")) - end - end - if verbose then - self:log ("%4.03fs: all done.\n", tt:get0()); - end - end - end -} - -prog:SubCommand { - name = "sched-params", - description = "Set/Get scheduler parameters at runtime.", - usage = "get or set ITEM=VAL", - - handler = function (self, arg) - local action = arg[1] - if action == "set" then - local params = arg[2] - if not params then - self:die ("No arguments specified for setting scheduler parameters.") - end - - local resp, err = f:rpc ("sched.params.set", { param = tostring(params) }) - if not resp then - if err == "Function not implemented" then - prog:die ("Scheduler parameters cannot be updated when scheduler is not loaded") - else - prog:die ("Unable to set scheduler parameters. %s\n", err) - end - end - else -- Else corresponding to the 'get' action, return all current values - local params = arg[2] - local resp, err = f:rpc ("sched.params.get", { }) - if not resp then - if err == "Function not implemented" then - prog:die ("Scheduler parameters cannot be queried when scheduler is not loaded") - else - prog:die ("Unable to obtain scheduler parameters. %s\n", err) - end - end - - for k,v in pairs(resp) do - if (k == "delay-sched") then -- Return a simple integer, but print true/false for consistency - local v_str = "" - if (v == 1) then - v = "true" - else - v = "false" - end - end - if not params then - print(k.."="..v) - elseif string.find(params,k,1,true) then -- We have strings with '-', so search for plain strings - print(k.."="..v) - end - end - end - end -} - --- Check for valid connection to flux: -if not f then prog:die ("Connecting to flux failed: %s\n", err) end - --- Process cmdline, run any subcommand, or exit with failure: -prog:run (arg) --- vi: ts=4 sw=4 expandtab - diff --git a/src/cmd/flux-wreckrun b/src/cmd/flux-wreckrun deleted file mode 100755 index d53829a49faa..000000000000 --- a/src/cmd/flux-wreckrun +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/env lua - -------------------------------------------------------------------------------- --- Modules: -------------------------------------------------------------------------------- -local flux = require 'flux' -local posix = require 'flux.posix' -local timer = require 'flux.timer' - -local prog = string.match (arg[0], "([^/]+)$") -local shortprog = prog:match ("flux%-(.+)$") - --- --- Termination state needs to remain a global for access from --- signal handler functions. See setup_signal_handlers() below. --- -terminate = false - -------------------------------------------------------------------------------- --- Local functions: -------------------------------------------------------------------------------- --- --- Check that parameter [a] is an integer -- -local function is_integer (a) - local b = tonumber (a) - return (type (b) == "number") and (math.floor(b) == b) -end - ---- --- Get the LWJ return code as highest of all task return codes ---- -local function lwj_return_code (f, wreck, id) - local max, msgs = wreck.status { flux = f, jobid = id } - local fn = getmetatable (wreck) [ max > 0 and "say" or "verbose" ] - for s,h in pairs (msgs) do - fn (wreck, "task%s %s: %s\n", #h > 1 and "s" or "", tostring (h:sort()), s) - end - return max -end - --- --- Helper for fake_R_lite(): generate a fake range string from 0 .. n, --- when n >= 1 --- -local function range_string (n) - if not n or n == 0 then return nil end - local s = "0" - if n > 1 then - s = s .. "-" .. n-1 - end - return s -end - --- --- Generate a fake R_lite representation for the current wreck request. --- If -g, --gpus-per-task was used on the command line, include GPUs. --- --- This assists with testing without sched or with the -I, --immediate flag. --- -local function fake_R_lite (f, wreck) - local ncores, ngpus = wreck.ncores, wreck.ngpus - local R_lite = {} - local nnodes = wreck.nnodes > 0 and wreck.nnodes or f.size - local cores_per_node = math.ceil (ncores / nnodes) - local gpus_per_node = ngpus and math.ceil (ngpus / nnodes) - - -- Allocate cores (and gpus) to R_lite in rank order until - -- no more cores are required - -- - for rank = 0, nnodes - 1 do - local corecnt = math.min (ncores, cores_per_node) - local gpucnt = ngpus and math.min (wreck.ngpus, gpus_per_node) - table.insert (R_lite, { rank = rank, - children = { - core = range_string (corecnt), - gpu = range_string (gpucnt) } - }) - ncores = ncores - corecnt - if ngpus then - ngpus = ngpus - gpucnt - end - if ncores == 0 then - return R_lite - end - end - return R_lite -end - -local function wreckrun_hooks_compile (wreck) - local code = wreck:getopt ("P") - local pre_launch_fn = nil - if code then - wreck:verbose ("compiling code '%s'\n", code) - pre_launch_fn, err = loadstring ("local wreck, lwj = ...; " .. code) - if not pre_launch_fn then - wreck:die ("--pre-launch-hook: %s\n", err) - end - end - -- Add a hooks table to existing 'wreck' object - wreck.hooks = { pre_launch = pre_launch_fn } - return true -end - -local function wreckrun_pre_launch ( wreck, lwj) - if wreck.hooks.pre_launch then - local r, v = pcall (wreck.hooks.pre_launch, wreck, lwj) - if not r then return nil, v end - end - return true -end - -------------------------------------------------------------------------------- --- Main program: -------------------------------------------------------------------------------- --- Open connection to flux broker: - -local f, err = flux.new () -if not f then - io.stderr:write ("Unable to connect to flux broker: "..err) - os.exit (1) -end - --- Parse cmdline args: --- -local wreck = require 'wreck' .new { prog = shortprog, flux = f } -local ioattach = require 'wreck'.ioattach -local logstream = require 'wreck'.logstream -local taskio -local log -local sigtimer -local state = "" - -wreck:add_options ({ - { name = 'pre-launch-hook', char = "P", arg = "CODE", - usage = "execute Lua CODE before launch." }, - { name = 'detach', char = "d", - usage = "Detach immediately after starting job" }, - { name = 'wait-until', char = "w", arg = "STATE", - usage = "Do not process stdio, but block until 'STATE'" }, - { name = 'immediate', char = "I", - usage = "Bypass scheduler and run immediately" }, -}) - -if not wreck:parse_cmdline (arg) then - wreck:die ("Failed to process cmdline args\n") -end - --- if nokz is in effect and the --output option is not used, AND --- either --detach or --wait-until are active, then store output by --- default in the kvs; o/w it is just dropped: -if (wreck.job_options.nokz and not wreck:getopt ('O')) and - (wreck:getopt ('d') or wreck:getopt ('w')) then - wreck.opts.O = "kvs://files.stdout" - if not wreck:getopt ('E') then wreck.opts.E = "kvs://files.stderr" end - - -- Regenerate wreck.output output file configuration: - wreck.output = wreck:output_file_config () -end - -wreckrun_hooks_compile (wreck) - --- Start in-program timer: -local tt = timer.new() - --- --- Create a new job in kvs --- -local jobid, err = wreck:createjob () -if not jobid then wreck:die ("%s\n", err) end - -wreck:verbose ("%4.03fs: Registered jobid %d\n", tt:get0(), jobid) - --- --- Get a handle to this lwj kvsdir: --- -local lwj, err = wreck:kvsdir (jobid) -if not lwj then wreck:die ("f:kvsdir(lwj.%d): %s\n", jobid, err) end - --- --- Check if job state is "complete" and all IO from tasks have closed: --- -local function check_job_completed () - if state == "failed" then - log:dump() - wreck:die ("job %d failed\n", jobid) - end - if (not taskio or taskio:complete()) and - (state == "completing" or state == "complete" or state == "reaped") then - local rc = lwj_return_code (f, wreck, jobid) - if rc == 0 then - wreck:verbose ("%.3fs: All tasks completed successfully.\n", - tt:get0 ()); - end - os.exit (rc) - end -end - ---- ---- Attach stdio kz streams and start all watchers ---- -local function wreck_attach_stdio (wreck) - -- - -- Open output to all tasks: - -- - taskio, err = ioattach { - flux = f, - jobid = jobid, - ntasks = wreck.ntasks, - labelio = wreck.opts.l, - nokz = wreck.job_options.nokz, - ioservices = wreck.ioservices, - on_completion = check_job_completed - } - if not taskio then wreck:die ("error opening task io: %s\n", err) end - - -- - -- Open stdin - -- - local kz, err = f:kz_open (tostring (lwj)..".input.files.stdin", "w") - if not kz then wreck:die (err) end - - -- - -- Add a file descriptor iowatcher for this script's stdin: - -- - local iow, err = f:iowatcher { - fd = posix.fileno (io.input()), - handler = function (iow, data) - if data.data then kz:write (data.data) end - if data.eof then kz:close () end - end - } - if not iow then wreck:die ("error opening stdin: %s\n", err) end - - return taskio -end - - --- --- Install watcher for this job's state before issuing any run request, --- (So that no state events are missed) --- -local kw, err = f:kvswatcher { - key = string.format ("%s.state", wreck:lwj_path (jobid)), - handler = function (kw, result) - if result then - state = result - wreck:verbose ("%-4.03fs: State = %s\n", tt:get0(), result) - if wreck.opts.w and state == wreck.opts.w then - os.exit (0) - end - check_job_completed () - end - end -} -if not kw then wreck:die ("kvs watch: %s\n", err) end - -local submitted = false -if not wreck:getopt ("I") then - -- Attempt to submit this existing job via submit-nocreate RPC: - -- - local rc, err = f:rpc ("job.submit-nocreate", { jobid = jobid }) - if rc then - submitted = true - wreck:verbose ("%-4.03fs: job.submit: Success\n", tt:get0()); - else - wreck:verbose ("%-4.03fs: job.submit: %s\n", tt:get0(), err) - end -end - -if not submitted then - -- - -- If submit failed due to lack of scheduler or use of the - -- -I, --immediate option, manually create R_lite and - --- send event to run the job. - -- - R_lite = fake_R_lite (f, wreck); - if not R_lite then wreck:die ("Failed to generate R_lite\n") end - lwj.R_lite = R_lite - lwj:commit() - - -- Ensure lwj nnodes matches fake allocation - lwj.nnodes = tonumber (wreck.nnodes) - lwj.ntasks = tonumber (wreck.ntasks) - - -- Execute any pre-launch hooks - local r, err = wreckrun_pre_launch (wreck, lwj) - if not r then - lwj.state = "failed" - lwj:commit() - wreck:die ("pre-launch-hook failed: %s", err) - end - - lwj:commit () - - -- Now send run event: - wreck:verbose ("%-4.03fs: Sending run event\n", tt:get0()) - local rc,err = f:sendevent ("wrexec.run.%d", jobid) - if not rc then wreck:die ("sendevent: %s\n", err) end - if wreck.opts.d then - print (jobid) - os.exit(0) - end -end - --- Only process stdio if no --wait-until option used -if not wreck.opts.w then - taskio = wreck_attach_stdio (wreck) -end - -log, err = logstream { - flux = f, - jobid = jobid, -} -if not log then wreck:die ("error opening log stream: %s\n", err) end - -local s, err = f:sighandler { - sigmask = { posix.SIGINT, posix.SIGTERM }, - handler = function (f, s, sig) - terminate = true - f:reactor_stop () - end -} - -local s, err = f:sighandler { - sigmask = { posix.SIGALRM }, - handler = function (f, s, sig) - wreck:say ("Killed by SIGALRM: state = %s\n", state) - os.exit (128+posix.SIGALRM) - end -} - --- --- Begin reactor loop: --- -local sigtimer = nil - -repeat - local r, err = f:reactor() - if not r then wreck:die ("flux_reactor: %s", err) end - -- - -- If we catch a signal then lwj:watch() will be interrupted. - -- Check to see if we should terminate the job now: - -- - if terminate then - wreck:verbose ("%4.03fs: Killing LWJ %d\n", tt:get0(), jobid) - local rc,err = f:sendevent ("wreck.%d.kill", jobid) - if not rc then - wreck:say ("Error: Failed to send kill event: %s", err) - end - if not sigtimer then - sigtimer = timer.new() - sigtimer:get() - elseif sigtimer:get() < 1.0 then - wreck:say ("Detaching from job. Processes may still be running\n"); - os.exit (0); - end - terminate = false - end -until false - - --- vi: ts=4 sw=4 expandtab diff --git a/src/cmd/flux.c b/src/cmd/flux.c index 0160e6044cd1..50c2622db733 100644 --- a/src/cmd/flux.c +++ b/src/cmd/flux.c @@ -194,11 +194,6 @@ int main (int argc, char *argv[]) flux_conf_get ("rc3_path", flags), 0); environment_from_env (env, "FLUX_PMI_LIBRARY_PATH", flux_conf_get ("pmi_library_path", flags), 0); - environment_from_env (env, "FLUX_WRECK_LUA_PATTERN", - flux_conf_get ("wreck_lua_pattern", flags), 0); - environment_from_env (env, "FLUX_WREXECD_PATH", - flux_conf_get ("wrexecd_path", flags), 0); - if ((flags & CONF_FLAG_INTREE)) environment_push (env, "FLUX_CONF_INTREE", "1"); diff --git a/src/common/Makefile.am b/src/common/Makefile.am index a0d6873c2b75..170628e010de 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -5,7 +5,6 @@ SUBDIRS = libtap \ libutil \ libflux \ libkvs \ - libjsc \ libjob \ libzio \ libcompat \ @@ -44,7 +43,6 @@ libflux_core_la_SOURCES = libflux_core_la_LIBADD = \ $(builddir)/libflux/libflux.la \ $(builddir)/libkvs/libkvs.la \ - $(builddir)/libjsc/libjsc.la \ $(builddir)/libjob/libjob.la \ $(builddir)/libsubprocess/libsubprocess.la \ libflux-internal.la diff --git a/src/common/core.h b/src/common/core.h index 6b671f3c0276..2293fb355dc0 100644 --- a/src/common/core.h +++ b/src/common/core.h @@ -17,7 +17,6 @@ #include "core/flux.h" #include "core/kvs.h" -#include "core/jstatctl.h" #include "core/job.h" #endif /* !_FLUX_CORE_H */ diff --git a/src/common/libflux-core.map b/src/common/libflux-core.map index c34072d193e0..78eddb1a1ada 100644 --- a/src/common/libflux-core.map +++ b/src/common/libflux-core.map @@ -1,6 +1,5 @@ { global: flux_*; - jsc_*; JSC_*; local: *; }; diff --git a/src/common/libflux/Makefile.am b/src/common/libflux/Makefile.am index 44848d4d8365..c735f5056f9d 100644 --- a/src/common/libflux/Makefile.am +++ b/src/common/libflux/Makefile.am @@ -24,8 +24,6 @@ installed_conf_cppflags = \ -DINSTALLED_PMI_LIBRARY_PATH=\"$(fluxlibdir)/libpmi.so\" \ -DINSTALLED_RC1_PATH=\"$(fluxrcdir)/rc1\" \ -DINSTALLED_RC3_PATH=\"$(fluxrcdir)/rc3\" \ - -DINSTALLED_WRECK_LUA_PATTERN=\"$(sysconfdir)/wreck/lua.d/*.lua\" \ - -DINSTALLED_WREXECD_PATH=\"$(fluxlibexecdir)/wrexecd\" \ -DINSTALLED_CMDHELP_PATTERN=\"${datadir}/flux/help.d/*.json\" \ -DINSTALLED_NO_DOCS_PATH=\"${datadir}/flux/.nodocs\" \ -DINSTALLED_RUNDIR=\"${runstatedir}/flux\" \ @@ -45,8 +43,6 @@ intree_conf_cppflags = \ -DINTREE_PMI_LIBRARY_PATH=\"$(abs_top_builddir)/src/common/.libs/libpmi.so\" \ -DINTREE_RC1_PATH=\"$(abs_top_srcdir)/etc/rc1\" \ -DINTREE_RC3_PATH=\"$(abs_top_srcdir)/etc/rc3\" \ - -DINTREE_WRECK_LUA_PATTERN=\"$(abs_top_srcdir)/src/modules/wreck/lua.d/*.lua\" \ - -DINTREE_WREXECD_PATH=\"$(abs_top_builddir)/src/modules/wreck/wrexecd\" \ -DINTREE_CMDHELP_PATTERN=\"${abs_top_builddir}/etc/flux/help.d/*.json\" \ -DINTREE_KEYDIR=\"${abs_top_builddir}/etc/flux\" \ -DINTREE_NO_DOCS_PATH=\"${abs_top_builddir}/etc/flux/.nodocs\" \ diff --git a/src/common/libflux/conf.c b/src/common/libflux/conf.c index 1cfc25d9ba2b..8ed3bedaf8b0 100644 --- a/src/common/libflux/conf.c +++ b/src/common/libflux/conf.c @@ -34,10 +34,6 @@ static struct config default_config[] = { { "cmdhelp_pattern",INSTALLED_CMDHELP_PATTERN, INTREE_CMDHELP_PATTERN }, { "pmi_library_path", INSTALLED_PMI_LIBRARY_PATH, INTREE_PMI_LIBRARY_PATH }, - { "wrexecd_path", INSTALLED_WREXECD_PATH, INTREE_WREXECD_PATH }, - { "wreck_lua_pattern", - INSTALLED_WRECK_LUA_PATTERN, - INTREE_WRECK_LUA_PATTERN }, { "keydir", NULL, INTREE_KEYDIR }, { "no_docs_path", INSTALLED_NO_DOCS_PATH, INTREE_NO_DOCS_PATH }, { "rundir", INSTALLED_RUNDIR, NULL }, diff --git a/src/common/libjsc/Makefile.am b/src/common/libjsc/Makefile.am deleted file mode 100644 index f786958e6e58..000000000000 --- a/src/common/libjsc/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -AM_CFLAGS = \ - $(WARNING_CFLAGS) \ - $(CODE_COVERAGE_CFLAGS) - -AM_LDFLAGS = \ - $(CODE_COVERAGE_LIBS) - -AM_CPPFLAGS = \ - -I$(top_srcdir) \ - -I$(top_srcdir)/src/include \ - -I$(top_builddir)/src/common/libflux \ - $(ZMQ_CFLAGS) $(JANSSON_CFLAGS) - -noinst_LTLIBRARIES = libjsc.la -fluxcoreinclude_HEADERS = jstatctl.h - -libjsc_la_SOURCES = \ - jstatctl.c diff --git a/src/common/libjsc/README.md b/src/common/libjsc/README.md deleted file mode 100644 index 0c5799ca0b39..000000000000 --- a/src/common/libjsc/README.md +++ /dev/null @@ -1,180 +0,0 @@ -Job Status and Control Application Programming Interface -=================== - -The Job Status and Control (JSC) API is a high-level abstraction that -allows its client software to monitor and control the status of Flux -jobs. It is designed to expose the job status and control abstraction -in a way to hide the underlying data layout of job information stored -within Flux's KVS data store. We expect that schedulers and runtime -tools will be its main users. This abstraction provides the producers -of job information including a task and program execution service -module such as `flux-wreckrun` with an opportunity to change and -optimize the data layout of jobs within the KVS without presenting -major impacts to the implementation of the schedulers and runtime tools. - -1. Design Consideration -------------- -The main design considerations are the following: ->1. Abstract out implementation-dependent job states; ->2. Provide flexible and easily extensible mechanisms to pass job -information; ->3. Use a minimalistic API set. - -The first consideration has led us to use a C enumerator (i.e., -*job\_state\_t*) to capture the job states. However, because Flux has -not yet defined its job schema, the second consideration discouraged us -to use a C user-defined type to pass job information with the client -software. Instead, JSC uses an JSON to capture the job information and -introduce the notion of Job Control Block (JCB) to have a structure on -this information. We will try to keep backward compatibility on JCB's -structure as we will extend this to keep abreast of the evolution of -Flux's job schema. Finally, the third consideration has led us to -introduce three simple API calls as the main JSC idioms: job -status-change notification as well as JCB query and update. Client -software can use the notification call to get the status of a job -asynchronously on a state change; the query and update calls to fetch -and update a job's JCB, respectively. - -2. Job States -------------- -The JSC API converts the state strings produced by `flux-wreckrun` and -the scheduler framework service comms module into a C enumerator: -*job\_state\_t*. Its elements are shown in Table 2-1. If the raw state -strings are changed in the future, one must accommodate this JCB -implementation accordingly--mapping the new strings to these state -elements. We expect the state set will further be extended as we will -add more advanced services such as elastic job and resource management -service. - -| Elements | Comment | -|--------------|------------------------------------------------------| -| J_NULL | No state has been assigned | -| J_RESERVED | Reserved in KVS | -| J_SUBMITTED | Added to KVS | -| J_PENDING | Pending | -| J_SCHEDREQ | Resource selection requested | -| J_ALLOCATED | Resource allocated/contained in the Flux instance | -| J_RUNREQUEST | Requested to be executed | -| J_STARTING | Starting | -| J_STOPPED | Stopped | -| J_RUNNING | Running | -| J_CANCELLED | Cancelled | -| J_COMPLETE | Complete | -| J_REAPED | Reaped to the upper-level Flux instance | -| J_FAILED | Failed before running | -| J_FOR\_RENT | To be extended | - -**Table 2-1** Job state elements - - -| Key | Macro | Value Type | Comment | -|------------|----------------|----------------|--------------------------------------------------------------------------------| -| jobid | JSC_JOBID | 64-bit integer | Job id | -| state-pair | JSC_STATE\_PAIR| dictionary | A dictionary containing this old and new states of the job. See Table 3-2. | -| rdesc | JSC_RDESC | dictionary | Information on the resources owned by this job. See Table 3-3. | -| rdl | JSC_RDL | string | RDL binary string allocated to the job | -| R_lite | JSC_R\_LITE | string | R\_lite serialized JSON string allocated to the job | - -**Table 3-1** Keys and values of top-level JCB attributes - - -| Key | Macro | Value Type | Comment | -|------------|-------------------------|----------------|---------------------------------------| -| ostate | JSC_STATE\_PAIR\_OSTATE | 64-bit integer | Old state (a *job\_state\_t* element) | -| nstate | JSC_STATE\_PAIR\_NSTATE | 64-bit integer | New state (a *job\_state\_t* element) | - -**Table 3-2** Keys and values of *state-pair* attribute - - -| Key | Macro | Value Type | Comment | -|------------|------------------|-----------------|---------------| -| nnodes | JSC_RDESC\_NNODES | 64-bit integer | Node count | -| ntasks | JSC_RDESC\_NTASKS | 64-bit integer | Process count | -| ncores | JSC_RDESC\_NCORES | 64-bit integer | core count | -| ngpus | JSC_RDESC\_NGPUS | 64-bit integer | GPU count | -| walltime | JSC_RDESC\_WALLTIME | 64-bit integer | Walltime | - -**Table 3-3** Keys and values of *rdesc* attribute - - -3. Job Control Block -------------- -Job Control Block (JCB) is our data schema containing the information -needed to manage a particular job. It contains information such as -jobid, resources owned by the job, as well as the processes spawned by -the job. The JSC API converts the raw information on a job into an -JCB, implemented as an JSON dictionary object. Our current JCB -structure is shown in Table 3-1 through 3-6. As Flux's job schema -evolves, we will extend JCB while trying our best to keep backward -compatibility. - - -4. Application Programming Interface -------------- -The Job Status and Control API currently provides three main calls: - -> Note: typedef int (\*jsc\_handler\_f)(json\_object \*base\_jcb, void -\*arg, int -errnum); -> ->- int jsc\_notify\_status (flux\_t h, jsc\_handler\_f callback, void -\*d); ->- int jsc\_query\_jcb (flux\_t h, int64\_t jobid, const char \*key, -char -\*\*jcb); ->- int jsc\_update\_jcb (flux\_t h, int64\_t jobid, const char \*key, -const char -\*jcb); - - - -#### jsc\_notify\_status -Register a *callback* to the asynchronous status change notification -service. *callback* will be invoked when the state of a job changes. -The *jobid* and *state-pair* as shown in Table 3-2 will be passed as -*base_jcb* into the *callback*. *d* is arbitrary data that will -transparently be passed into *callback*. However, one should pass its -*flux_t* object as part of this callback data. Note that the caller -must start its reactor to get an asynchronous status change -notification via *callback*. This is because it uses the KVS-watch -facility which has the same limitation. One can register multiple -callbacks by calling this function multiple times. The callbacks will -be invoked in the order they are registered. Returns 0 on success; -otherwise -1. - - -#### jsc\_query\_jcb -Query the *key* attribute of JCB of *jobid*. The JCB info on this -attribute will be passed via *jcb*. It is the responsibility of the -caller to release *jcb*. All ownership associated with the -sub-attributes in *jcb*'s hierarchy are transferred to *jcb*, so that -json_object_put (\*jcb) will free this hierarchy in its entirety. -Returns 0 on success; otherwise -1. - -#### jsc\_update\_jcb -Update the *key* attribute within the JCB of *jobid*. The top-level -attribute of *jcb* should be the same as *key*. Returns 0 on success; -otherwise -1. This will not release *jcb* so it is the responsibility -of the caller to free *jcb*. - ->**Notes:** - ->1. JCB granularity optimization -- one can optimize the amounts of JCB -information piggybacked with each notification (*base_jcb*). One can -further extend the single attribute-wise query/update pattern to -group-wise ones once the access patterns of JCS API's clients are -known. - -5. Testing -------------- - -To facilitate the testing of this API, we created an utility command: -`flux-jstat`. Its usage is the following: - - Usage: flux-jstat notify - flux-jstat query jobid - flux-jstat update jobid - -Further, `flux-core/t/t2001-jsc.t` contains various test cases that use -this utility. - diff --git a/src/common/libjsc/jstatctl.c b/src/common/libjsc/jstatctl.c deleted file mode 100644 index 21fa98881d42..000000000000 --- a/src/common/libjsc/jstatctl.c +++ /dev/null @@ -1,862 +0,0 @@ -/************************************************************\ - * Copyright 2015 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#include "jstatctl.h" -#include "src/common/libutil/oom.h" -#include "src/common/libutil/log.h" -#include "src/common/libutil/xzmalloc.h" -#include "src/common/libutil/iterators.h" -#include "src/common/libutil/lru_cache.h" - -const char *const JSC_JOBID = "jobid"; -const char *const JSC_STATE_PAIR = "state-pair"; -const char *const JSC_STATE_PAIR_OSTATE = "ostate"; -const char *const JSC_STATE_PAIR_NSTATE = "nstate"; -const char *const JSC_RDESC = "rdesc"; -const char *const JSC_RDESC_NNODES = "nnodes"; -const char *const JSC_RDESC_NTASKS = "ntasks"; -const char *const JSC_RDESC_NCORES = "ncores"; -const char *const JSC_RDESC_NGPUS = "ngpus"; -const char *const JSC_RDESC_WALLTIME = "walltime"; -const char *const JSC_RDL = "rdl"; -const char *const JSC_R_LITE = "R_lite"; - -/******************************************************************************* - * * - * Internal User Define Types and Data * - * * - *******************************************************************************/ - -typedef struct { - job_state_t i; - const char *s; -} stab_t; - -typedef struct { - jsc_handler_f cb; - void *arg; -} cb_pair_t; - -typedef struct { - zhash_t *active_jobs; - lru_cache_t *kvs_paths; - flux_msg_handler_t **handlers; - zlist_t *callbacks; - flux_t *h; -} jscctx_t; - -static stab_t job_state_tab[] = { - { J_NULL, "null" }, - { J_RESERVED, "reserved" }, - { J_SUBMITTED, "submitted" }, - { J_PENDING, "pending" }, - { J_SCHEDREQ, "schedreq" }, - { J_SELECTED, "selected" }, - { J_ALLOCATED, "allocated" }, - { J_RUNREQUEST, "runrequest" }, - { J_STARTING, "starting" }, - { J_SYNC, "sync" }, - { J_RUNNING, "running" }, - { J_CANCELLED, "cancelled" }, - { J_COMPLETING, "completing" }, - { J_COMPLETE, "complete" }, - { J_REAPED, "reaped" }, - { J_FAILED, "failed" }, - { J_FOR_RENT, "for_rent" }, - { -1, NULL }, -}; - - -/****************************************************************************** - * * - * Utilities * - * * - ******************************************************************************/ - -const char *jsc_job_num2state (job_state_t i) -{ - stab_t *ss = job_state_tab; - while (ss->s != NULL) { - if (ss->i == i) - return ss->s; - ss++; - } - return NULL; -} - -int jsc_job_state2num (const char *s) -{ - stab_t *ss = job_state_tab; - while (ss->s != NULL) { - if (!strcmp (ss->s, s)) - return ss->i; - ss++; - } - return -1; -} - -static void freectx (void *arg) -{ - jscctx_t *ctx = arg; - zhash_destroy (&(ctx->active_jobs)); - lru_cache_destroy (ctx->kvs_paths); - zlist_destroy (&(ctx->callbacks)); - flux_msg_handler_delvec (ctx->handlers); - free (ctx); -} - -static jscctx_t *getctx (flux_t *h) -{ - jscctx_t *ctx = (jscctx_t *)flux_aux_get (h, "jstatctrl"); - if (!ctx) { - ctx = xzmalloc (sizeof (*ctx)); - if (!(ctx->active_jobs = zhash_new ())) - oom (); - if (!(ctx->kvs_paths = lru_cache_create (256))) - oom (); - lru_cache_set_free_f (ctx->kvs_paths, free); - if (!(ctx->callbacks = zlist_new ())) - oom (); - ctx->h = h; - if (flux_aux_set (h, "jstatctrl", ctx, freectx) < 0) - oom (); - } - return ctx; -} - -static int lwj_kvs_path (flux_t *h, int64_t id, char **pathp) -{ - int rc = -1; - const char *path; - flux_future_t *f; - uint32_t nodeid = FLUX_NODEID_ANY; - - if (!(f = flux_rpc_pack (h, "job.kvspath", nodeid, 0, "{s:[I]}", "ids", id)) - || flux_rpc_get_unpack (f, "{s:[s]}", "paths", &path) < 0) - goto out; - if (!(*pathp = strdup (path))) - goto out; - rc = 0; -out: - flux_future_destroy (f); - return (rc); -} - -static int jscctx_add_jobid_path (jscctx_t *ctx, int64_t id, const char *path) -{ - char *s; - char key [21]; - memset (key, 0, sizeof (key)); - if (sprintf (key, "%"PRId64, id) < 0) - return (-1); - if (!(s = strdup (path))) - return (-1); - if (lru_cache_put (ctx->kvs_paths, key, s) < 0) - free (s); - return (0); -} - -static const char * jscctx_jobid_path (jscctx_t *ctx, int64_t id) -{ - char *path; - char key [21]; - - memset (key, 0, sizeof (key)); - if (sprintf (key, "%"PRId64, id) < 0) - return (NULL); - if ((path = lru_cache_get (ctx->kvs_paths, key))) - return path; - if (lwj_kvs_path (ctx->h, id, &path) < 0) - return (NULL); - if (lru_cache_put (ctx->kvs_paths, key, path) < 0) - return (NULL); - return (path); -} - -static int *active_insert (zhash_t *hash, int64_t jobid) -{ - char key[32]; - int *state; - snprintf (key, sizeof (key), "%"PRId64, jobid); - if (!(state = calloc (1, sizeof (int)))) - return NULL; - if (zhash_insert (hash, key, state) < 0) { - free (state); - return NULL; - } - zhash_freefn (hash, key, (zhash_free_fn *)free); - return state; -} - -static int *active_lookup (zhash_t *hash, int64_t jobid) -{ - char key[32]; - snprintf (key, sizeof (key), "%"PRId64, jobid); - return zhash_lookup (hash, key); -} - -static void active_delete (zhash_t *hash, int64_t jobid) -{ - char key[32]; - snprintf (key, sizeof (key), "%"PRId64, jobid); - return zhash_delete (hash, key); -} - -/****************************************************************************** - * * - * Internal JCB Accessors * - * * - ******************************************************************************/ - -static int jobid_exist (flux_t *h, int64_t jobid) -{ - jscctx_t *ctx = getctx (h); - flux_future_t *f = NULL; - const char *path; - int rc = -1; - - if (!active_lookup (ctx->active_jobs, jobid)) { - if (!(path = jscctx_jobid_path (ctx, jobid))) - goto done; - if (!(f = flux_kvs_lookup (h, NULL, FLUX_KVS_READDIR, path)) - || flux_future_get (f, NULL) < 0) - goto done; - } - rc = 0; -done: - flux_future_destroy (f); - return rc; -} - -static char *lwj_vkey (flux_t *h, int64_t jobid, const char *fmt, va_list ap) -{ - jscctx_t *ctx = getctx (h); - const char *base; - char *name; - char *key; - - if (!(base = jscctx_jobid_path (ctx, jobid))) - return (NULL); - if (vasprintf (&name, fmt, ap) < 0) - return (NULL); - if (asprintf (&key, "%s%s", base, name) < 0) { - free (name); - return (NULL); - } - free (name); - return (key); -} - -static char * __attribute__ ((format (printf, 3, 4))) -lwj_key (flux_t *h, int64_t jobid, const char *fmt, ...) -{ - va_list ap; - char *key; - - va_start (ap, fmt); - key = lwj_vkey (h, jobid, fmt, ap); - va_end (ap); - return key; -} - -static flux_future_t * __attribute__ ((format (printf, 3, 4))) -lookup_job_attribute (flux_t *h, int64_t jobid, const char *fmt, ...) -{ - va_list ap; - flux_future_t *f; - char *key; - - va_start (ap, fmt); - key = lwj_vkey (h, jobid, fmt, ap); - va_end (ap); - if (!key) - return NULL; - f = flux_kvs_lookup (h, NULL, 0, key); - free (key); - return f; -} - -/* Old state is unavailable through the query. - * One should use notification service instead. - */ -static json_t *query_state_pair (flux_t *h, int64_t jobid) -{ - json_t *jcb = NULL; - char *key; - int64_t state; - const char *state_str; - flux_future_t *f = NULL; - - if (!(key = lwj_key (h, jobid, ".state"))) - return NULL; - - if (!(f = flux_kvs_lookup (h, NULL, 0, key)) - || flux_kvs_lookup_get_unpack (f, "s", &state_str) < 0) - goto done; - state = jsc_job_state2num (state_str); - if (!(jcb = json_pack ("{s:{s:I s:I}}", - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, state, - JSC_STATE_PAIR_NSTATE, state))) - goto done; -done: - flux_future_destroy (f); - free (key); - return jcb; -} - -static json_t *query_rdesc (flux_t *h, int64_t jobid) -{ - json_t *jcb = NULL; - int64_t nnodes; - int64_t ntasks; - int64_t ncores; - int64_t ngpus; - int64_t walltime; - - if (jsc_query_rdesc_efficiently (h, jobid, &nnodes, &ntasks, &ncores, - &walltime, &ngpus) < 0) - return NULL; - - if (!(jcb = json_pack ("{s:{s:I s:I s:I s:I s:I}}", - JSC_RDESC, - JSC_RDESC_NNODES, nnodes, - JSC_RDESC_NTASKS, ntasks, - JSC_RDESC_NCORES, ncores, - JSC_RDESC_WALLTIME, walltime, - JSC_RDESC_NGPUS, ngpus))) - return NULL; - return jcb; -} - -int jsc_query_rdesc_efficiently (flux_t *h, int64_t jobid, - int64_t *nnodes, int64_t *ntasks, - int64_t *ncores, int64_t *walltime, - int64_t *ngpus) -{ - flux_future_t *f_nnodes = NULL; - flux_future_t *f_ntasks = NULL; - flux_future_t *f_ncores = NULL; - flux_future_t *f_wall = NULL; - flux_future_t *f_ngpus = NULL; - int rc = -1; - - /* Send requests (in parallel) - */ - if (nnodes && !(f_nnodes = lookup_job_attribute (h, jobid, ".nnodes"))) - goto error; - if (ntasks && !(f_ntasks = lookup_job_attribute (h, jobid, ".ntasks"))) - goto error; - if (ncores && !(f_ncores = lookup_job_attribute (h, jobid, ".ncores"))) - goto error; - if (walltime && !(f_wall = lookup_job_attribute (h, jobid, ".walltime"))) - goto error; - if (ngpus && !(f_ngpus = lookup_job_attribute (h, jobid, ".ngpus"))) - goto error; - - /* Handle responses - */ - if (f_nnodes && flux_kvs_lookup_get_unpack (f_nnodes, "I", nnodes) < 0) - goto error; - if (f_ntasks && flux_kvs_lookup_get_unpack (f_ntasks, "I", ntasks) < 0) - goto error; - if (f_ncores && flux_kvs_lookup_get_unpack (f_ncores, "I", ncores) < 0) - goto error; - if (f_wall && flux_kvs_lookup_get_unpack (f_wall, "I", walltime) < 0) - goto error; - if (f_ngpus && flux_kvs_lookup_get_unpack (f_ngpus, "I", ngpus) < 0) - goto error; - - rc = 0; -error: - flux_future_destroy (f_nnodes); - flux_future_destroy (f_ntasks); - flux_future_destroy (f_ncores); - flux_future_destroy (f_wall); - flux_future_destroy (f_ngpus); - return rc; -} - -static json_t *query_rdl (flux_t *h, int64_t jobid) -{ - flux_future_t *f; - const char *json_str; - json_t *rdl; - json_t *jcb = NULL; - - if (!(f = lookup_job_attribute (h, jobid, ".rdl"))) - return NULL; - if (flux_kvs_lookup_get (f, &json_str) < 0) - goto error; - if (!(rdl = json_loads (json_str, 0, NULL))) - goto error; - if (!(jcb = json_pack ("{s:o}", - JSC_RDL, rdl))) { - json_decref (rdl); - goto error; - } -error: - flux_future_destroy (f); - return jcb; -} - -static json_t *query_r_lite (flux_t *h, int64_t jobid) -{ - flux_future_t *f; - const char *json_str; - json_t *r_lite; - json_t *jcb = NULL; - - if (!(f = lookup_job_attribute (h, jobid, ".R_lite"))) - return NULL; - if (flux_kvs_lookup_get (f, &json_str) < 0) - goto error; - if (!(r_lite = json_loads (json_str, 0, NULL))) - goto error; - if (!(jcb = json_pack ("{s:o}", - JSC_R_LITE, r_lite))) { - json_decref (r_lite); - goto error; - } -error: - flux_future_destroy (f); - return jcb; -} - -static int send_state_event (flux_t *h, job_state_t st, int64_t jobid) -{ - flux_msg_t *msg = NULL; - char *topic = NULL; - int rc = -1; - - if (asprintf (&topic, "jsc.state.%s", jsc_job_num2state (st)) < 0) - goto done; - if ((msg = flux_event_pack (topic, "{ s:I }", "jobid", jobid)) == NULL) - goto done; - if (flux_send (h, msg, 0) < 0) - goto done; - rc = 0; -done: - flux_msg_destroy (msg); - free (topic); - return rc; -} - -static int update_state (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - int state; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - - if (json_unpack (jcb, "{s:{s:i}}", - JSC_STATE_PAIR, - JSC_STATE_PAIR_NSTATE, &state) < 0) - goto done; - if ((state >= J_FOR_RENT) || (state < J_NULL)) - goto done; - if (!(key = lwj_key (h, jobid, ".state"))) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "s", jsc_job_num2state (state)) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL,0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - if (send_state_event (h, state, jobid) < 0) - goto done; - rc = 0; -done: - flux_future_destroy (f); - flux_kvs_txn_destroy (txn); - free (key); - return rc; -} - -static int update_rdesc (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - int64_t nnodes; - int64_t ntasks; - int64_t ncores; - int64_t walltime; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - - if (json_unpack (jcb, "{s:{s:I s:I s:I s:I}}", - JSC_RDESC, - JSC_RDESC_NNODES, &nnodes, - JSC_RDESC_NTASKS, &ntasks, - JSC_RDESC_NCORES, &ncores, - JSC_RDESC_WALLTIME, &walltime) < 0) - goto done; - if ((nnodes < 0) || (ntasks < 0) || (ncores < 0) || (walltime < 0)) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (!(key = lwj_key (h, jobid, ".nnodes"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", nnodes) < 0) - goto done; - free (key); - if (!(key = lwj_key (h, jobid, ".ntasks"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", ntasks) < 0) - goto done; - free (key); - if (!(key = lwj_key (h, jobid, ".walltime"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", walltime) < 0) - goto done; - free (key); - if (!(key = lwj_key (h, jobid, ".ncores"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", ncores) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL, 0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - rc = 0; -done: - flux_future_destroy (f); - free (key); - flux_kvs_txn_destroy (txn); - return rc; -} - -static int update_rdl (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - json_t *rdl; - - if (json_unpack (jcb, "{s:o}", JSC_RDL, &rdl) < 0) - goto done; - if (!(key = lwj_key (h, jobid, ".rdl"))) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "O", rdl) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL, 0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - rc = 0; -done: - flux_kvs_txn_destroy (txn); - flux_future_destroy (f); - free (key); - return rc; -} - -static int update_r_lite (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - json_t *r_lite; - - if (json_unpack (jcb, "{s:o}", JSC_R_LITE, &r_lite) < 0) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (!(key = lwj_key (h, jobid, ".R_lite"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "O", r_lite) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL, 0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - rc = 0; -done: - flux_kvs_txn_destroy (txn); - flux_future_destroy (f); - free (key); - return rc; -} - -static json_t *get_update_jcb (flux_t *h, int64_t jobid, const char *state_str) -{ - jscctx_t *ctx = getctx (h); - int *state; - int nstate = jsc_job_state2num (state_str); - int ostate = J_FOR_RENT; - json_t *o; - - if ((state = active_lookup (ctx->active_jobs, jobid))) { - ostate = *state; - if (nstate == J_COMPLETE || nstate == J_FAILED) - active_delete (ctx->active_jobs, jobid); - else - *state = nstate; - } - if (!(o = json_pack ("{s:I s:{s:i s:i}}", JSC_JOBID, jobid, - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, ostate, - JSC_STATE_PAIR_NSTATE, nstate))) - oom (); - return o; -} - - -/****************************************************************************** - * * - * Internal Asynchronous Notification Mechanisms * - * * - ******************************************************************************/ - -static int invoke_cbs (flux_t *h, int64_t j, json_t *jcb, int errnum) -{ - int rc = 0; - cb_pair_t *c = NULL; - jscctx_t *ctx = getctx (h); - for (c = zlist_first (ctx->callbacks); c; c = zlist_next (ctx->callbacks)) { - char *jcb_str = json_dumps (jcb, 0); - if (!jcb_str) - oom (); - if (c->cb (jcb_str, c->arg, errnum) < 0) - rc = -1; - free (jcb_str); - } - return rc; -} - -static json_t *get_reserve_jcb (flux_t *h, int64_t jobid) -{ - jscctx_t *ctx = getctx (h); - json_t *jcb; - int ostate = J_NULL; - int nstate = J_RESERVED; - int *state; - - if (!(jcb = json_pack ("{s:I s{s:i s:i}}", - JSC_JOBID, jobid, - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, ostate, - JSC_STATE_PAIR_NSTATE, nstate))) - goto error; - - if (!(state = active_insert (ctx->active_jobs, jobid))) - goto error; - *state = nstate; - return jcb; - -error: - json_decref (jcb); - return NULL; -} - -static json_t *get_submit_jcb (flux_t *h, const flux_msg_t *msg, int64_t jobid) -{ - jscctx_t *ctx = getctx (h); - int64_t ntasks; - int64_t nnodes; - int64_t ncores; - int64_t ngpus; - int64_t walltime; - int ostate = J_NULL; - int nstate = J_SUBMITTED; - int *state; - json_t *jcb = NULL; - - if (flux_event_unpack (msg, NULL, "{ s:I s:I s:I s:I s:I }", - "ntasks", &ntasks, - "nnodes", &nnodes, - "ncores", &ncores, - "ngpus", &ngpus, - "walltime", &walltime) < 0) - goto error; - if (!(jcb = json_pack ("{s:I s:{s:i s:i} s:{s:I s:I s:I s:I s:I}}", - JSC_JOBID, jobid, - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, ostate, - JSC_STATE_PAIR_NSTATE, nstate, - JSC_RDESC, - JSC_RDESC_NNODES, nnodes, - JSC_RDESC_NTASKS, ntasks, - JSC_RDESC_NCORES, ncores, - JSC_RDESC_NGPUS, ngpus, - JSC_RDESC_WALLTIME, walltime))) - goto error; - - if (!(state = active_lookup (ctx->active_jobs, jobid))) { - if (!(state = active_insert (ctx->active_jobs, jobid))) - goto error; - } - *state = nstate; - - return jcb; -error: - json_decref (jcb); - return NULL; -} - -static bool job_is_finished (const char *state) -{ - if (strcmp (state, jsc_job_num2state (J_COMPLETE)) == 0 - || strcmp (state, jsc_job_num2state (J_FAILED)) == 0) - return true; - return false; -} - -static void job_state_cb (flux_t *h, flux_msg_handler_t *mh, - const flux_msg_t *msg, void *arg) -{ - jscctx_t *ctx = getctx (h); - json_t *jcb = NULL; - int64_t jobid; - const char *topic = NULL; - const char *state = NULL; - const char *kvs_path = NULL; - int len = 12; - - if (flux_event_unpack (msg, &topic, "{s:I s?:s}", - "jobid", &jobid, - "kvs_path", &kvs_path) < 0) - goto done; - if (kvs_path && jscctx_add_jobid_path (ctx, jobid, kvs_path) < 0) - goto done; - - if (strncmp (topic, "jsc", 3) == 0) - len = 10; - - state = topic + len; - if (strcmp (state, jsc_job_num2state (J_RESERVED)) == 0) - jcb = get_reserve_jcb (h, jobid); - else if (strcmp (state, jsc_job_num2state (J_SUBMITTED)) == 0) - jcb = get_submit_jcb (h, msg, jobid); - else - jcb = get_update_jcb (h, jobid, state); - - (void)invoke_cbs (h, jobid, jcb, (jcb)? 0 : EINVAL); - - if (job_is_finished (state)) - active_delete (ctx->active_jobs, jobid); -done: - json_decref (jcb); - return; -} - -/****************************************************************************** - * * - * Public Job Status and Control API * - * * - ******************************************************************************/ -static const struct flux_msg_handler_spec htab[] = { - { FLUX_MSGTYPE_EVENT, "wreck.state.*", job_state_cb, 0 }, - { FLUX_MSGTYPE_EVENT, "jsc.state.*", job_state_cb, 0 }, - FLUX_MSGHANDLER_TABLE_END -}; - -int jsc_notify_status (flux_t *h, jsc_handler_f func, void *d) -{ - int rc = -1; - cb_pair_t *c = NULL; - jscctx_t *ctx = NULL; - flux_msg_handler_t **handlers; - - if (!func) - goto done; - if (flux_event_subscribe (h, "wreck.state.") < 0) - goto done; - if (flux_event_subscribe (h, "jsc.state.") < 0) - goto done; - if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0) - goto done; - - ctx = getctx (h); - ctx->handlers = handlers; - c = (cb_pair_t *) xzmalloc (sizeof(*c)); - c->cb = func; - c->arg = d; - if (zlist_append (ctx->callbacks, c) < 0) - goto done; - zlist_freefn (ctx->callbacks, c, free, true); - rc = 0; - -done: - return rc; -} - -int jsc_query_jcb (flux_t *h, int64_t jobid, const char *key, char **jcb_str) -{ - json_t *jcb = NULL; - - if (!key) - return -1; - if (jobid_exist (h, jobid) < 0) - return -1; - - if (!strcmp (key, JSC_JOBID)) - jcb = json_pack ("{s:I}", JSC_JOBID, jobid); - else if (!strcmp (key, JSC_STATE_PAIR)) - jcb = query_state_pair (h, jobid); - else if (!strcmp (key, JSC_RDESC)) - jcb = query_rdesc (h, jobid); - else if (!strcmp (key, JSC_RDL)) - jcb = query_rdl (h, jobid); - else if (!strcmp (key, JSC_R_LITE)) - jcb = query_r_lite (h, jobid); - - if (!jcb) - return -1; - char *s = json_dumps (jcb, 0); - if (!s) { - json_decref (jcb); - return -1; - } - *jcb_str = s; - return 0; -} - -int jsc_update_jcb (flux_t *h, int64_t jobid, const char *key, - const char *jcb_str) -{ - int rc = -1; - json_t *jcb; - - if (!jcb_str || !(jcb = json_loads (jcb_str, 0, NULL))) - return -1; - if (jobid_exist (h, jobid) != 0) - goto done; - if (!strcmp (key, JSC_STATE_PAIR)) - rc = update_state (h, jobid, jcb); - else if (!strcmp (key, JSC_RDESC)) - rc = update_rdesc (h, jobid, jcb); - else if (!strcmp (key, JSC_RDL)) - rc = update_rdl (h, jobid, jcb); - else if (!strcmp (key, JSC_R_LITE)) - rc = update_r_lite (h, jobid, jcb); -done: - json_decref (jcb); - return rc; -} - -/* - * vi: ts=4 sw=4 expandtab - */ diff --git a/src/common/libjsc/jstatctl.h b/src/common/libjsc/jstatctl.h deleted file mode 100644 index 6d3e1ab0b558..000000000000 --- a/src/common/libjsc/jstatctl.h +++ /dev/null @@ -1,136 +0,0 @@ -/************************************************************\ - * Copyright 2015 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#ifndef _FLUX_CORE_JSTATCTRL_H -#define _FLUX_CORE_JSTATCTRL_H 1 - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Define the job states (an abstraction independent of - * underlying task and program execution services (RFC 8) - * and scheduler implementation details (e.g., how the - * attributes of a job are stored in KVS.) For more details, - * please refer to README.md - */ -typedef enum { - J_NULL = 0, /*!< The state has yet to be assigned */ - - /* WRECK job initial condition states: - */ - J_RESERVED = 1, /*!< Reserved by the program execution service */ - J_SUBMITTED = 2, /*!< Submitted to the system */ - - /* Scheduler internal states: - */ - J_PENDING = 11, /*!< Pending */ - J_SCHEDREQ = 12, /*!< Resources requested to be selected */ - J_SELECTED = 13, /*!< Assigned to requested resource in RDL */ - J_ALLOCATED = 14, /*!< Got alloc/contained by the program exec service */ - - /* WRECK job execution states: - */ - J_RUNREQUEST= 21, /*!< Requested to be executed */ - J_STARTING = 22, /*!< Starting */ - J_SYNC = 23, /*!< Tasks stopped in exec waiting for a tool */ - J_RUNNING = 24, /*!< Running */ - J_COMPLETING= 26, /*!< Completing (all tasks exited, epilog running) */ - - /* WRECK job terminal states: - */ - J_CANCELLED = 51, /*!< Cancelled (before execution) */ - J_COMPLETE = 52, /*!< Completed */ - J_FAILED = 53, /*!< Failed (before exec) */ - - /* Scheduler post exec states: - */ - J_REAPED = 101, /*!< Reaped */ - J_FOR_RENT = 102, /*!< Space For Rent */ -} job_state_t; - -typedef int (*jsc_handler_f)(const char *base_jcb, void *arg, int errnum); - -/* TODO: find a better way to manage this hierarchical - * JCB attributes space - */ - -extern const char *const JSC_JOBID; -extern const char *const JSC_STATE_PAIR; -extern const char *const JSC_STATE_PAIR_OSTATE; -extern const char *const JSC_STATE_PAIR_NSTATE; -extern const char *const JSC_RDESC; -extern const char *const JSC_RDESC_NNODES; -extern const char *const JSC_RDESC_NTASKS; -extern const char *const JSC_RDESC_NCORES; -extern const char *const JSC_RDESC_NGPUS; -extern const char *const JSC_RDESC_WALLTIME; -extern const char *const JSC_RDL; -extern const char *const JSC_R_LITE; - -/** - * Register a callback to the asynchronous status change notification service. - * "callback" will be invoked when the state of a job changes. The "jobid" - * and "state-pair" will be passed as "base_jcb" into the callback. - * "d" is arbitrary data that will transparently be passed into "callback." - * However, one should pass its flux_t object as part of this callback data. - * Note that the caller must start its reactor to get an asynchronous status - * change notification via "callback." - * One can register mutliple callbacks by calling this function - * multiple times. The callbacks will be invoked in the order - * they are registered. Returns 0 on success; otherwise -1. - */ -int jsc_notify_status (flux_t *h, jsc_handler_f callback, void *d); - -/** - * Query the "key" attribute of JCB of "jobid." The JCB info on this attribute - * will be passed via "jcb." It is the caller's responsibility to free "jcb." - * Returns 0 on success; otherwise -1. - */ -int jsc_query_jcb (flux_t *h, int64_t jobid, const char *key, char **jcb); - - -/** - * Update the "key" attribute of the JCB of "jobid". The top-level attribute - * of "jcb" should be identical to "key". Return 0 on success; otherwise -1. - * This will not release "jcb," so it is the caller's responsibility to - * free "jcb." - */ -int jsc_update_jcb (flux_t *h, int64_t jobid, const char *key, const char *jcb); - - -/** - * A convenience routine (returning the internal state name correponding to "s.") - */ -const char *jsc_job_num2state (job_state_t s); -int jsc_job_state2num (const char *s); - -/** - * Accessor for nnodes, ntasks, ncores, walltime that circumvents - * JSON encode/decode. N.B. this is a workaround for performance issue - * encountered when scheduling high throughput workloads. - */ -int jsc_query_rdesc_efficiently (flux_t *h, int64_t jobid, - int64_t *nnodes, int64_t *ntasks, - int64_t *ncores, int64_t *walltime, - int64_t *ngpus); - -#ifdef __cplusplus -} -#endif - -#endif /*! _FLUX_CORE_JSTATCTRL_H */ - -/* - * vi: ts=4 sw=4 expandtab - */ diff --git a/src/include/flux/core.h b/src/include/flux/core.h index 539a5130c08a..65a80a5ecfa7 100644 --- a/src/include/flux/core.h +++ b/src/include/flux/core.h @@ -16,7 +16,6 @@ #include "src/common/libflux/flux.h" #include "src/common/libkvs/kvs.h" -#include "src/common/libjsc/jstatctl.h" #include "src/common/libjob/job.h" #endif /* FLUX_CORE_H */ diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am index 57e2457b88f9..defc952933d3 100644 --- a/src/modules/Makefile.am +++ b/src/modules/Makefile.am @@ -4,7 +4,6 @@ SUBDIRS = \ kvs \ kvs-watch \ content-sqlite \ - wreck \ cron \ aggregator \ userdb \ diff --git a/src/modules/wreck/.gitignore b/src/modules/wreck/.gitignore deleted file mode 100644 index ea6dea2a1cb6..000000000000 --- a/src/modules/wreck/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/wrexecd diff --git a/src/modules/wreck/Makefile.am b/src/modules/wreck/Makefile.am deleted file mode 100644 index 6d2f89ae2359..000000000000 --- a/src/modules/wreck/Makefile.am +++ /dev/null @@ -1,97 +0,0 @@ -AM_CFLAGS = \ - $(WARNING_CFLAGS) \ - $(CODE_COVERAGE_CFLAGS) - -AM_LDFLAGS = \ - $(CODE_COVERAGE_LIBS) - -AM_CPPFLAGS = \ - -I$(top_srcdir) \ - -I$(top_srcdir)/src/include \ - -I$(top_builddir)/src/common/libflux \ - -I$(top_srcdir)/src/common/liblsd \ - $(LUA_INCLUDE) \ - $(ZMQ_CFLAGS) - -wreckscriptsdir = $(sysconfdir)/wreck/lua.d - -# -# Comms module -# -fluxmod_LTLIBRARIES = \ - job.la - -fluxlibexec_PROGRAMS = \ - wrexecd - -fluxmod_libadd = $(top_builddir)/src/common/libflux-core.la \ - $(top_builddir)/src/common/libflux-internal.la - -noinst_LTLIBRARIES = \ - librcalc.la -librcalc_la_SOURCES = \ - rcalc.c \ - rcalc.h - -job_la_SOURCES = job.c wreck_job.c wreck_job.h -job_la_LDFLAGS = $(AM_LDFLAGS) $(fluxmod_ldflags) -module -job_la_LIBADD = $(fluxmod_libadd) librcalc.la - -wrexecd_SOURCES = \ - wrexecd.c \ - luastack.c \ - luastack.h - -wrexecd_libs = \ - $(top_builddir)/src/bindings/lua/libfluxlua.la \ - $(top_builddir)/src/common/libflux-internal.la \ - $(top_builddir)/src/common/libpmi/libpmi.la \ - $(top_builddir)/src/common/libflux-core.la \ - $(top_builddir)/src/common/libflux-optparse.la - -wrexecd_LDFLAGS = \ - $(AM_LDFLAGS) -export-dynamic - -wrexecd_LDADD = \ - librcalc.la \ - $(wrexecd_libs) \ - $(ZMQ_LIBS) $(LUA_LIB) $(LIBPTHREAD) $(HWLOC_LIBS) - -TESTS = \ - test_wreck_job.t - -test_ldadd = \ - $(top_builddir)/src/common/libflux-internal.la \ - $(top_builddir)/src/common/libflux-core.la \ - $(top_builddir)/src/common/libtap/libtap.la \ - $(ZMQ_LIBS) $(LIBPTHREAD) - -test_cppflags = \ - $(AM_CPPFLAGS) \ - -I$(top_srcdir)/src/common/libtap - -check_PROGRAMS = $(TESTS) - -test_wreck_job_t_SOURCES = test/wreck_job.c -test_wreck_job_t_CPPFLAGS = $(test_cppflags) -test_wreck_job_t_LDADD = \ - $(top_builddir)/src/modules/wreck/wreck_job.o \ - $(test_ldadd) - -dist_wreckscripts_SCRIPTS = \ - lua.d/01-env.lua \ - lua.d/timeout.lua \ - lua.d/output.lua \ - lua.d/epilog.lua \ - lua.d/input.lua \ - lua.d/mvapich.lua \ - lua.d/pmi-mapping.lua \ - lua.d/intel_mpi.lua \ - lua.d/openmpi.lua \ - lua.d/spectrum.lua \ - lua.d/cuda_devices.lua - -# XXX: Hack below to force rebuild of unbuilt wrexecd dependencies -# -$(wrexecd_libs): - @cd `dirname $@` && $(MAKE) `basename $@` diff --git a/src/modules/wreck/job.c b/src/modules/wreck/job.c deleted file mode 100644 index 328817e269b3..000000000000 --- a/src/modules/wreck/job.c +++ /dev/null @@ -1,939 +0,0 @@ -/************************************************************\ - * Copyright 2014 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include - -#include -#include -#include -#include -#if WITH_TCMALLOC -#if HAVE_GPERFTOOLS_HEAP_PROFILER_H - #include -#elif HAVE_GOOGLE_HEAP_PROFILER_H - #include -#else - #error gperftools headers not configured -#endif -#endif /* WITH_TCMALLOC */ - -#include - -#include - -#include "src/common/libutil/log.h" -#include "src/common/libutil/fdwalk.h" -#include "src/common/libsubprocess/subprocess.h" - -#include "rcalc.h" -#include "wreck_job.h" - -#define MAX_JOB_PATH 1024 - -/* - * lwj directory hierarchy parameters: - * XXX: should be in module context struct perhaps, but - * this all needs to go away soon anyway, so... - * - * directory levels is the number of parent directories - * (e.g. 3 would result in lwj-active.x.y.z.jobid, - * 0 is lwj.jobid) - * - * bits_per_directory is the number of prefix bits to use - * for each parent directory, results in 2^bits entries - * per subdirectory, except for the top-level which will - * grow without bound (well up to 64bit lwj id values). - * - * These values can be set as broker attrs during flux-start, - * e.g. flux start -o,-Swreck.lwj-dir-levels=3 - * -o,-Swreck.lwj-bits-per-dir=8 - */ -static int kvs_dir_levels = 2; -static int kvs_bits_per_dir = 7; - -uint32_t broker_rank; -const char *local_uri = NULL; - -zhash_t *active_jobs = NULL; - -/* - * Return as 64bit integer the portion of integer `n` - * masked from bit position `a` to position `b`, - * then subsequently shifted by `a` bits (to keep - * numbers small). - */ -static inline uint64_t prefix64 (uint64_t n, int a, int b) -{ - uint64_t mask = ((-1ULL >> (64 - b)) & ~((1ULL << a) - 1)); - return ((n & mask) >> a); -} - -/* - * Convert lwj id to kvs path under `lwj-active` using a kind of - * prefix hiearchy of max levels `levels`, using `bits_per_dir` bits - * for each directory. Returns a kvs key path or NULL on failure. - */ -static char * lwj_to_path (uint64_t id, int levels, int bits_per_dir) -{ - char buf [MAX_JOB_PATH] = "lwj"; - int len = 3; - int nleft = sizeof (buf) - len; - int i, n; - - /* Build up kvs directory from lwj. down */ - for (i = levels; i > 0; i--) { - int b = bits_per_dir * i; - uint64_t d = prefix64 (id, b, b + bits_per_dir); - if ((n = snprintf (buf+len, nleft, ".%"PRIu64, d)) < 0 || n > nleft) - return NULL; - len += n; - nleft -= n; - } - n = snprintf (buf+len, sizeof (buf) - len, ".%"PRIu64, id); - if (n < 0 || n > nleft) - return NULL; - return (strdup (buf)); -} - -static char * id_to_path (uint64_t id) -{ - return (lwj_to_path (id, kvs_dir_levels, kvs_bits_per_dir)); -} - -static flux_future_t *next_jobid (flux_t *h) -{ - flux_future_t *f; - - f = flux_rpc_pack (h, "seq.fetch", 0, 0, "{s:s,s:i,s:i,s:b}", - "name", "lwj", - "preincrement", 1, - "postincrement", 0, - "create", true); - return f; -} - -static int next_jobid_get (flux_future_t *f, int64_t *id) -{ - int64_t jobid; - if (flux_rpc_get_unpack (f, "{s:I}", "value", &jobid) < 0) - return -1; - *id = jobid; - return 0; -} - -static char * realtime_string (char *buf, size_t sz) -{ - struct timespec tm; - clock_gettime (CLOCK_REALTIME, &tm); - memset (buf, 0, sz); - snprintf (buf, sz, "%ju.%06ld", (uintmax_t) tm.tv_sec, tm.tv_nsec/1000); - return (buf); -} - -/* Publish wreck.state. event. - * Future is fulfilled once event has received a sequence number. - * This ensures that response to job create request is not sent until this - * happens. See issue #337. - */ -static flux_future_t *send_create_event (flux_t *h, struct wreck_job *job) -{ - char topic[64]; - flux_future_t *f; - int flags = 0; - - if (snprintf (topic, sizeof (topic), "wreck.state.%s", job->state) - >= sizeof (topic)) { - errno = EINVAL; - return NULL; - } - if (!(f = flux_event_publish_pack (h, topic, flags, - "{s:I,s:s,s:i,s:i,s:i,s:i,s:i}", - "jobid", job->id, - "kvs_path", job->kvs_path, - "ntasks", job->ntasks, - "ncores", job->ncores, - "nnodes", job->nnodes, - "ngpus", job->ngpus, - "walltime", job->walltime))) - return NULL; - return f; -} - -static int add_jobinfo_txn (flux_kvs_txn_t *txn, struct wreck_job *job) -{ - char buf [64]; - const char *json_str; - char key[MAX_JOB_PATH]; - size_t keysz = sizeof (key); - flux_msg_t *msg = wreck_job_get_aux (job); - - const char *k; - json_t *v; - json_t *o = NULL; - - if (flux_request_decode (msg, NULL, &json_str) < 0) - goto error; - if (!json_str || !(o = json_loads (json_str, 0, NULL))) - goto inval; - if (snprintf (key, sizeof (key), "%s.state", job->kvs_path) >= sizeof (key)) - goto inval; - if (flux_kvs_txn_pack (txn, 0, key, "s", job->state) < 0) - goto error; - - json_object_foreach (o, k, v) { - int rc; - char *s; - if (snprintf (key, keysz, "%s.%s", job->kvs_path, k) >= keysz) - goto inval; - if (!(s = json_dumps (v, JSON_COMPACT|JSON_ENCODE_ANY))) - goto error; - rc = flux_kvs_txn_put (txn, 0, key, s); - free (s); - if (rc < 0) - goto error; - } - if (snprintf (key, sizeof (key), "%s.create-time", job->kvs_path) - >= sizeof (key)) - goto inval; - if (flux_kvs_txn_pack (txn, 0, key, "s", - realtime_string (buf, sizeof (buf))) < 0) - goto error; - json_decref (o); - return 0; -inval: - errno = EINVAL; -error: - json_decref (o); - return -1; -} - -static bool ping_sched (flux_t *h) -{ - bool retval = false; - flux_future_t *f; - if (!(f = flux_rpc_pack (h, "sched.ping", 0, 0, "{s:i}", "seq", 0))) { - flux_log_error (h, "ping_sched"); - goto out; - } - if (flux_future_get (f, NULL) >= 0) - retval = true; -out: - flux_future_destroy (f); - return (retval); -} - -static bool sched_loaded (flux_t *h) -{ - static bool v = false; - if (!v && ping_sched (h)) - v = true; - return (v); -} - -/* - * Respond to job.submit-nocreate rpc, which allows a job previously - * created by "job.create" to be submitted (The "job.submit" rpc - * creates *and* submits a job). Set the job state to "submitted" - * and issue the wreck.state.submitted event with appropriate payload. - * - * This rpc is used by flux-wreckrun when a scheduler is loaded so that - * "interactive" jobs are subject to normal scheduling. - * (The -I, --immediate flag of flux-wreckrun can be used to disable - * this behavior) - */ -static void job_submit_only (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - int64_t jobid; - struct wreck_job *job = NULL; - flux_future_t *f = NULL;; - - if (!sched_loaded (h)) { - errno = ENOSYS; - goto error; - } - if (flux_request_unpack (msg, NULL, "{s:I}", "jobid", &jobid) < 0) - goto error; - - if (!(job = wreck_job_lookup (jobid, active_jobs))) { - errno = ENOENT; - goto error; - } - wreck_job_set_state (job, "submitted"); - if (!(f = send_create_event (h, job))) - goto error; - if (flux_future_get (f, NULL) < 0) - goto error; - if (flux_respond_pack (h, msg, "{s:I}", "jobid", job->id) < 0) - flux_log_error (h, "flux_respond"); - flux_future_destroy (f); - return; -error: - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "flux_respond"); - flux_future_destroy (f); -} - -/* Handle request to broadcast wreck.state. event. - * This concludes the continuation chain started at job_create_cb(). - * Respond to the original request and destroy 'job'. - */ -static void job_create_event_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - flux_msg_t *msg = wreck_job_get_aux (job); - - if (flux_future_get (f, NULL) < 0) { - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - } - else { - if (flux_respond_pack (h, msg, "{s:I,s:s,s:s}", - "jobid", job->id, - "state", job->state, - "kvs_path", job->kvs_path) < 0) - flux_log_error (h, "flux_respond_pack"); - } - flux_future_destroy (f); -} - - -/* Handle KVS commit response, then send request to broadcast - * wreck.state. event. - * Function is continued in job_create_event_continuation(). - */ -static void job_create_kvs_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - flux_msg_t *msg = wreck_job_get_aux (job); - flux_future_t *f_next = NULL; - - if (flux_future_get (f, NULL) < 0) - goto error; - - /* Preemptively insert this job into the active job hash on this - * node, making it available for use by job_submit_only (). We do - * this *before* we send the event so we avoid racing with the - * event handler that also inserts active jobs. - */ - if (wreck_job_insert (job, active_jobs) < 0) - goto error; - if (!(f_next = send_create_event (h, job))) - goto error; - if (flux_future_then (f_next, -1., job_create_event_continuation, job) < 0) - goto error; - flux_future_destroy (f); - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - flux_future_destroy (f_next); - flux_future_destroy (f); -} - -/* Handle next available jobid response, then issue KVS commit request - * to write job data to KVS. - * Function is continued in job_create_kvs_continuation(). - */ -static void job_create_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - flux_msg_t *msg = wreck_job_get_aux (job); - flux_kvs_txn_t *txn = NULL; - flux_future_t *f_next = NULL; - - if (next_jobid_get (f, &job->id) < 0) - goto error; - if (!(job->kvs_path = id_to_path (job->id))) - goto error; - if (!(txn = flux_kvs_txn_create ())) - goto error; - if (add_jobinfo_txn (txn, job) < 0) - goto error; - if (!(f_next = flux_kvs_commit (h, NULL, 0, txn))) - goto error; - if (flux_future_then (f_next, -1., job_create_kvs_continuation, job) < 0) - goto error; - flux_log (h, LOG_DEBUG, "Setting job %lld to %s", (long long)job->id, - job->state); - flux_kvs_txn_destroy (txn); - flux_future_destroy (f); - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - flux_kvs_txn_destroy (txn); - flux_future_destroy (f_next); - flux_future_destroy (f); - wreck_job_destroy (job); -} - -/* Handle job.create and job.submit requests. - * Create 'job', then send request for next available jobid. - * Function is continued in job_create_continuation(). - */ -static void job_create_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - const char *topic; - flux_msg_t *cpy; - struct wreck_job *job; - flux_future_t *f = NULL; - - if (!(job = wreck_job_create ())) - goto error; - if (flux_request_unpack (msg, &topic, "{s?:i s?:i s?:i s?:i s?:i}", - "ntasks", &job->ntasks, - "nnodes", &job->nnodes, - "ncores", &job->ncores, - "ngpus", &job->ngpus, - "walltime", &job->walltime) < 0) - goto error; - if (!(cpy = flux_msg_copy (msg, true))) - goto error; - wreck_job_set_aux (job, cpy, (flux_free_f)flux_msg_destroy); - if (strcmp (topic, "job.create") == 0) - wreck_job_set_state (job, "reserved"); - else if (strcmp (topic, "job.submit") == 0) { - if (!sched_loaded (h)) { - errno = ENOSYS; - goto error; - } - wreck_job_set_state (job, "submitted"); - } - if (!(f = next_jobid (h))) - goto error; - if (flux_future_then (f, -1., job_create_continuation, job) < 0) - goto error; - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - wreck_job_destroy (job); - flux_future_destroy (f); -} - -static json_t *json_id_to_json_path (flux_t *h, json_t *value) -{ - int64_t id; - char *path = NULL; - json_t *o = NULL; - int errnum = EPROTO; - - if (!json_is_integer (value) || (id = json_integer_value (value)) < 0) - goto out; - if (!(path = id_to_path (id))) { - errnum = errno; - flux_log (h, LOG_ERR, "kvspath_cb: lwj_to_path failed"); - goto out; - } - if (!(o = json_string (path))) { - errnum = errno; - flux_log_error (h, "kvspath_cb: json_string"); - goto out; - } - errnum = 0; -out: - free (path); - errno = errnum; - return (o); -} - -static void job_kvspath_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - int errnum = EPROTO; - size_t index; - json_t *id_list = NULL; - json_t *paths = NULL; - json_t *value; - - if ((flux_request_unpack (msg, NULL, "{s:o}", "ids", &id_list) < 0) - || !json_is_array (id_list)) { - flux_log_error (h, "kvspath_cb failed to unpack message"); - goto out; - } - - paths = json_array (); - json_array_foreach (id_list, index, value) { - json_t *o = json_id_to_json_path (h, value); - if (o == NULL) { - errnum = errno; - goto out; - } - if (json_array_append_new (paths, o) < 0) { - errnum = errno; - flux_log_error (h, "kvspath_cb: json_array_append_new"); - json_decref (o); - goto out; - } - } - if (flux_respond_pack (h, msg, "{s:O}", "paths", paths) < 0) - flux_log_error (h, "kvspath_cb: flux_respond_pack"); - errnum = 0; -out: - if (errnum && flux_respond (h, msg, errnum, NULL) < 0) - flux_log_error (h, "kvspath_cb: flux_respond"); - json_decref (paths); -} - -static int flux_attr_set_int (flux_t *h, const char *attr, int val) -{ - char buf [16]; - int n = snprintf (buf, sizeof (buf), "%d", val); - if (n < 0 || n >= sizeof (buf)) - return (-1); - return flux_attr_set (h, attr, buf); -} - -static int flux_attr_get_int (flux_t *h, const char *attr, int *valp) -{ - long n; - const char *tmp; - char *p; - - if ((tmp = flux_attr_get (h, attr)) == NULL) - return (-1); - n = strtoul (tmp, &p, 10); - if (n == LONG_MAX) - return (-1); - if ((p == tmp) || (*p != '\0')) { - errno = EINVAL; - return (-1); - } - *valp = (int) n; - return (0); -} - -static void completion_cb (flux_subprocess_t *p) -{ - flux_t *h; - struct wreck_job *job = NULL; - int tmp; - - h = flux_subprocess_aux_get (p, "handle"); - job = flux_subprocess_aux_get (p, "job"); - - assert (h && job); - - /* skip output text if exit code == 0 */ - if (!(tmp = flux_subprocess_exit_code (p))) - goto cleanup; - - if (tmp > 0) { - flux_log_error (h, "job%ju: wrexecd: Exit %d", - (uintmax_t) job->id, tmp); - } - else if (tmp < 0) { - if ((tmp = flux_subprocess_signaled (p)) < 0) - flux_log_error (h, "job%ju: unknown exit status", (uintmax_t) job->id); - else - flux_log_error (h, "job%ju: wrexecd: %s", - (uintmax_t) job->id, strsignal (tmp)); - - } - -cleanup: - wreck_job_destroy (job); - flux_subprocess_destroy (p); -} - -static void state_change_cb (flux_subprocess_t *p, flux_subprocess_state_t state) -{ - flux_t *h; - struct wreck_job *job; - - h = flux_subprocess_aux_get (p, "handle"); - job = flux_subprocess_aux_get (p, "job"); - - assert (h && job); - - // XXX: Update job state to failed - if (state == FLUX_SUBPROCESS_EXEC_FAILED) { - flux_log_error (h, "spawn: job%ju: wrexecd exec failure", (uintmax_t) job->id); - flux_subprocess_destroy (p); - } - else if (state == FLUX_SUBPROCESS_FAILED) { - flux_log_error (h, "spawn: job%ju: wrexecd failure", (uintmax_t) job->id); - flux_subprocess_destroy (p); - } -} - -static void io_cb (flux_subprocess_t *p, const char *stream) -{ - int lenp = 0; - const char *ptr; - - if ((ptr = flux_subprocess_read_line (p, stream, &lenp)) - && lenp > 0) { - flux_t *h; - struct wreck_job *job; - int level = LOG_INFO; - - h = flux_subprocess_aux_get (p, "handle"); - job = flux_subprocess_aux_get (p, "job"); - - assert (h && job); - - if (!strcasecmp (stream, "STDERR")) - level = LOG_ERR; - - flux_log (h, level, - "job%ju: wrexecd says: %s", - (uintmax_t) job->id, ptr); - } -} - -static flux_cmd_t *wrexecd_cmd_create (flux_t *h, struct wreck_job *job) -{ - flux_cmd_t *cmd = NULL; - const char *wrexecd_path; - char *cwd = NULL; - char buf [4096]; - int n; - - if (!(cmd = flux_cmd_create (0, NULL, NULL))) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_create"); - goto error; - } - if (!(wrexecd_path = flux_attr_get (h, "wrexec.wrexecd_path"))) { - flux_log_error (h, "wrexecd_cmd_create: flux_attr_get"); - goto error; - } - if (flux_cmd_argv_append (cmd, "%s", wrexecd_path) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_argv_append"); - goto error; - } - n = snprintf (buf, sizeof(buf), "--lwj-id=%ju", (uintmax_t) job->id); - if ((n >= sizeof (buf)) || (n < 0)) { - flux_log_error (h, "failed to append id to cmdline for job%ju\n", - (uintmax_t) job->id); - goto error; - } - if (flux_cmd_argv_append (cmd, "%s", buf) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_argv_append"); - goto error; - } - n = snprintf (buf, sizeof (buf), "--kvs-path=%s", job->kvs_path); - if ((n >= sizeof (buf)) || (n < 0)) { - flux_log_error (h, "failed to append kvspath to cmdline for job%ju\n", - (uintmax_t) job->id); - goto error; - } - if (flux_cmd_argv_append (cmd, "%s", buf) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_argv_append"); - goto error; - } - /* flux_rexec() requires cwd to be set */ - if (!(cwd = get_current_dir_name ())) { - flux_log_error (h, "wrexecd_cmd_create: get_current_dir_name"); - goto error; - } - if (flux_cmd_setcwd (cmd, cwd) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_setcwd"); - goto error; - } - - free (cwd); - return (cmd); -error: - free (cwd); - flux_cmd_destroy (cmd); - return (NULL); -} - -static int spawn_exec_handler (flux_t *h, struct wreck_job *job) -{ - flux_cmd_t *cmd = NULL; - flux_subprocess_t *p = NULL; - flux_subprocess_ops_t ops = { - .on_completion = completion_cb, - .on_state_change = state_change_cb, - .on_channel_out = NULL, - .on_stdout = io_cb, - .on_stderr = io_cb - }; - - if (!(cmd = wrexecd_cmd_create (h, job))) { - flux_log_error (h, "wrexecd_cmd_create"); - goto error; - } - - if (!(p = flux_rexec (h, FLUX_NODEID_ANY, 0, cmd, &ops))) { - flux_log_error (h, "flux_rexec"); - goto error; - } - - if (flux_subprocess_aux_set (p, "handle", h, NULL) < 0) { - flux_log_error (h, "flux_subprocess_aux_set"); - goto error; - } - - if (flux_subprocess_aux_set (p, "job", job, NULL) < 0) { - flux_log_error (h, "flux_subprocess_aux_set"); - goto error; - } - - /* Take a reference on this job since it is now embedded in - * a flux rexec. - */ - flux_cmd_destroy (cmd); - wreck_job_incref (job); - return (0); - -error: - flux_cmd_destroy (cmd); - flux_subprocess_destroy (p); - return (-1); -} - -static bool Rlite_targets_this_node (flux_t *h, const char *key, - const char *R_lite) -{ - rcalc_t *r = NULL; - bool result; - - if (!(r = rcalc_create (R_lite))) { - if (broker_rank == 0) - flux_log (h, LOG_ERR, "Unable to parse %s", key); - return false; - } - result = rcalc_has_rank (r, broker_rank); - rcalc_destroy (r); - return result; -} - -/* Handle response to lookup of R_lite. If this node is targetted, - * spawn wrexecd. If R_lite doesn't exist, fallback to old method - * of looking up rank.N, with one more continuation. - */ -static void runevent_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - const char *key = flux_kvs_lookup_get_key (f); - const char *R_lite; - - if (flux_kvs_lookup_get (f, &R_lite) < 0) { - if (broker_rank == 0) - flux_log (h, LOG_INFO, "No %s: %s", key, flux_strerror (errno)); - goto done; - } - if (!Rlite_targets_this_node (h, key, R_lite)) - goto done; - - if (spawn_exec_handler (h, job) < 0) - goto done; -done: - flux_future_destroy (f); -} - -static int64_t id_from_tag (const char *tag) -{ - unsigned long l; - char *p; - errno = 0; - l = strtoul (tag, &p, 10); - if (*p != '\0') - return (-1); - if (l == 0 && errno == EINVAL) - return (-1); - else if (l == ULONG_MAX && errno == ERANGE) - return (-1); - return l; -} - -/* Handle wrexec.run. event. - * Determine if assigned resources are on this broker rank, then spawn - * wrexecd if so. This function sends request to read R_lite, - * then continues in runevent_continuation(). - */ -static void runevent_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, - void *arg) -{ - int64_t id; - const char *topic; - struct wreck_job *job; - flux_future_t *f = NULL; - char k[MAX_JOB_PATH]; - - if (flux_event_decode (msg, &topic, NULL) < 0) - goto error; - id = id_from_tag (topic+11); - if (!(job = wreck_job_lookup (id, active_jobs))) { - errno = ENOENT; - goto error; - } - if (!job->kvs_path) - goto error; - if (snprintf (k, sizeof (k), "%s.R_lite", job->kvs_path) >= sizeof (k)) { - errno = EINVAL; - goto error; - } - if (!(f = flux_kvs_lookup (h, NULL, 0, k))) - goto error; - if (flux_future_then (f, -1., runevent_continuation, job) < 0) - goto error; - /* N.B. 'f' and 'job' are destroyed by runevent_continuation() */ - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - flux_future_destroy (f); -} - -/* Track job state transition in active_jobs hash - * Currently only id, kvs_path, and state are tracked. - */ -static void wreck_state_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - int64_t id; - const char *topic; - const char *kvs_path = NULL; - struct wreck_job *job; - - if (flux_event_unpack (msg, &topic, "{s:I s?s}", - "jobid", &id, - "kvs_path", &kvs_path) < 0) - goto error; - topic += 12; // state comes after "wreck.state." (12 chars) - if (strlen (topic) == 0 || strlen (topic) >= sizeof (job->state)) { - errno = EPROTO; - goto error; - } - if (!(job = wreck_job_lookup (id, active_jobs))) { - if (!(job = wreck_job_create ())) - goto error; - job->id = id; - if (kvs_path && !(job->kvs_path = strdup (kvs_path))) - goto error_destroy; - if (wreck_job_insert (job, active_jobs) < 0) - goto error_destroy; - } - wreck_job_set_state (job, topic); - if ( !strcmp (job->state, "complete") - || !strcmp (job->state, "failed") - || !strcmp (job->state, "cancelled")) - wreck_job_delete (id, active_jobs); - return; -error_destroy: - wreck_job_destroy (job); -error: - flux_log_error (h, "%s", __FUNCTION__); -} - -static void job_list_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - char *json_str = NULL; - int max = 0; - const char *include = NULL; - const char *exclude = NULL; - - if (flux_request_unpack (msg, NULL, "{s?:i s?:s s?:s}", - "max", &max, - "include", &include, - "exclude", &exclude) < 0) - goto error; - if (!(json_str = wreck_job_list (active_jobs, max, include, exclude))) - goto error; - if (flux_respond (h, msg, 0, json_str) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - free (json_str); - return; -error: - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); -} - -static const struct flux_msg_handler_spec mtab[] = { - { FLUX_MSGTYPE_REQUEST, "job.create", job_create_cb, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.submit", job_create_cb, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.submit-nocreate", job_submit_only, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.kvspath", job_kvspath_cb, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.list", job_list_cb, 0 }, - { FLUX_MSGTYPE_EVENT, "wrexec.run.*", runevent_cb, 0 }, - { FLUX_MSGTYPE_EVENT, "wreck.state.*", wreck_state_cb, 0 }, - FLUX_MSGHANDLER_TABLE_END -}; - -int mod_main (flux_t *h, int argc, char **argv) -{ - flux_msg_handler_t **handlers = NULL; - int rc = -1; - - if (!(active_jobs = zhash_new ())) { - flux_log_error (h, "zhash_new"); - return (-1); - } - if (flux_msg_handler_addvec (h, mtab, NULL, &handlers) < 0) { - flux_log_error (h, "flux_msg_handler_addvec"); - goto done; - } - if ((flux_event_subscribe (h, "wrexec.run.") < 0)) { - flux_log_error (h, "flux_event_subscribe"); - goto done; - } - if ((flux_event_subscribe (h, "wreck.state.") < 0)) { - flux_log_error (h, "flux_event_subscribe"); - goto done; - } - - if ((flux_attr_get_int (h, "wreck.lwj-dir-levels", &kvs_dir_levels) < 0) - && (flux_attr_set_int (h, "wreck.lwj-dir-levels", kvs_dir_levels) < 0)) { - flux_log_error (h, "failed to get or set lwj-dir-levels"); - goto done; - } - if ((flux_attr_get_int (h, "wreck.lwj-bits-per-dir", - &kvs_bits_per_dir) < 0) - && (flux_attr_set_int (h, "wreck.lwj-bits-per-dir", - kvs_bits_per_dir) < 0)) { - flux_log_error (h, "failed to get or set lwj-bits-per-dir"); - goto done; - } - - if (flux_get_rank (h, &broker_rank) < 0) { - flux_log_error (h, "flux_get_rank"); - goto done; - } - - if (!(local_uri = flux_attr_get (h, "local-uri"))) { - flux_log_error (h, "flux_attr_get (\"local-uri\")"); - goto done; - } - - if (flux_reactor_run (flux_get_reactor (h), 0) < 0) { - flux_log_error (h, "flux_reactor_run"); - goto done; - } - rc = 0; -done: - flux_msg_handler_delvec (handlers); - zhash_destroy (&active_jobs); - return rc; -} - -MOD_NAME ("job"); - -/* - * vi: ts=4 sw=4 expandtab - */ diff --git a/src/modules/wreck/lua.d/01-env.lua b/src/modules/wreck/lua.d/01-env.lua deleted file mode 100644 index 2de1374a20bf..000000000000 --- a/src/modules/wreck/lua.d/01-env.lua +++ /dev/null @@ -1,57 +0,0 @@ - --- --- Set WRECK environment from environment table [environ] --- -local function rexec_set_env (environ) - if not environ then return end - - local env = wreck.environ - for k,v in pairs (environ) do - env[k] = v - end -end --- --- Set common environment and working directory for all tasks --- -function rexecd_init () - local posix = require 'flux.posix' - - -- Initialize environment with top level lwj.environ - local k = wreck.flux:kvsdir("lwj") - rexec_set_env (k.environ) - - -- Get kvsdir handle to this LWJ's entry in KVS and override - -- with lwj..environ - -- - local lwj = wreck.kvsdir - rexec_set_env (lwj.environ) - - -- Check for current working directory as lwj..cwd - local cwd = lwj.cwd - if cwd then - posix.chdir (cwd) - end -end - --- --- Per-task version of above: --- If lwj..environ or cwd exist then these override settings --- from lwj.{environ,cwd}: --- -function rexecd_task_init () - local posix = require 'flux.posix' - local taskid = wreck.taskid; - - local task = wreck.by_task - if not task then return end - - -- Set any task-specific environ from lwj...environ - rexec_set_env (task.environ) - - -- Each task can have a different cwd: - local cwd = task.cwd - if cwd then - posix.chdir (cwd) - end -end --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/cuda_devices.lua b/src/modules/wreck/lua.d/cuda_devices.lua deleted file mode 100644 index ea5da88837fd..000000000000 --- a/src/modules/wreck/lua.d/cuda_devices.lua +++ /dev/null @@ -1,75 +0,0 @@ -local gpubind = wreck:getopt ("gpubind") -if gpubind == "no" or gpubind == "off" then - return -end - --- Set CUDA_VISIBLE_DEVICES for all tasks on any rank with one or --- more "gpu" resources - -local gpuinfo = {} -function gpuinfo_create (wreck, gpus) - local g = {} - -- Use affinity.cpuset as a convenience to parse the GPU list, which - -- is in nodeset form (e.g. "0-1" or "0,2-5", etc.) - -- - local gset, err = require 'flux.affinity'.cpuset.new (gpus) - if not gset then - wreck:log_error ("Unable to parse GPU list [%s]: %s", gpus, err) - return nil - end - local g = { - gpuids = gset:expand (), - ngpus = gset:count (), - ntasks = wreck.tasks_per_node [wreck.nodeid] - } - - -- If per-task binding is requested, ensure ngpus is evenly divisible - -- into ntasks: - if gpubind == "per-task" and g.ngpus % g.ntasks == 0 then - g.ngpus_per_task = g.ngpus/g.ntasks - end - return g -end - -function rexecd_init () - -- NB: Lua arrays are indexed starting at 1, so this rank's index - -- into R_lite rank array is nodeid + 1: - -- - local index = wreck.nodeid + 1 - - -- Grab local resources structure from kvs for this nodeid: - -- - local Rlocal = wreck.kvsdir.R_lite[index].children - - -- If a gpu resource list is set for this rank, then expand it and - -- set CUDA_VISIBLE_DEVICES to the result: - -- - local gpus = Rlocal.gpu - if not gpus then return end - - gpuinfo = gpuinfo_create (wreck, gpus) - -- If ngpus_per_task is not set, then set CUDA_VISIBLE_DEVICES the same - -- for all tasks: - if not gpuinfo.ngpus_per_task then - local ids = table.concat (gpuinfo.gpuids, ",") - wreck.environ ["CUDA_VISIBLE_DEVICES"] = ids - end - -- Always set CUDA_DEVICE_ORDER=PCI_BUS_ID to ensure CUDA ids match - -- IDs known to flux scheduler. - wreck.environ ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" -end - -function rexecd_task_init () - -- If ngpus_per_task is set, then select that many GPUs from the gpuids - -- list assigned to this rank for the current task: - if not gpuinfo.ngpus_per_task then return end - - local basis = gpuinfo.ngpus_per_task * wreck.taskid - local t = {} - for i = 1,gpuinfo.ngpus_per_task do - table.insert (t, gpuinfo.gpuids [basis + i]) - end - wreck.environ ["CUDA_VISIBLE_DEVICES"] = table.concat (t, ",") -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/epilog.lua b/src/modules/wreck/lua.d/epilog.lua deleted file mode 100644 index c7a32b41b8a8..000000000000 --- a/src/modules/wreck/lua.d/epilog.lua +++ /dev/null @@ -1,21 +0,0 @@ -local posix = require 'flux.posix' - --- execute a path from kvs `key` -local function run_kvs (key) - local epilog = wreck.kvsdir [key] or wreck.flux:kvs_get ("lwj."..key) - if not epilog then return end - return os.execute (epilog) -end - -function rexecd_complete () - local rc, err = run_kvs ("epilog.pre") - if not rc then wreck:log_msg ("error: epilog: %s", err) end -end - --- rexecd_exit callback happens after the job is in the complete state -function rexecd_exit () - local rc, err = run_kvs ("epilog.post") - if not rc then wreck:log_msg ("error: epilog.post: %s", err) end -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/input.lua b/src/modules/wreck/lua.d/input.lua deleted file mode 100644 index 83efcc38ef53..000000000000 --- a/src/modules/wreck/lua.d/input.lua +++ /dev/null @@ -1,202 +0,0 @@ --- --- Process any stdin configuration for a job, opening files --- if requested, and setting up task stdin links as --- directed in lwj..input.config --- -local posix = require 'flux.posix' - --- temporary "set" class until we have nodeset lua bindings -local taskset = {} -function taskset:__index (key) - if tonumber (key) then - return self.t [tostring(key)] - end - return rawget (taskset,key) -end -function taskset.new (arg) - local hostlist = require 'flux.hostlist' - local t = {} - local hl = arg and hostlist.new ("["..arg.."]") or hostlist.new () - for i in hl:next () do - t[tostring(i)] = true - end - return setmetatable ({ t = t }, taskset) -end -function taskset:len () - local count = 0 - for i,v in pairs (self.t) do - count = count + 1 - end - return count -end - -function taskset:set (i) - local k = tostring (i) - local prev = self.t[k] - self.t[k] = true - return prev or false -end - -function taskset:clear (i) - local k = tostring (i) - local prev = self.t[k] - self.t[k] = nil - return prev or false -end - -function taskset:iterator () - return pairs (self.t) -end - ----- - -local function kvs_input_config (wreck) - local o = wreck.kvsdir ["input.config"] - if not o then - o = wreck.flux:kvsdir ("lwj.input.config") - end - return o -end - -local inputconf = {} -inputconf.__index = inputconf -function inputconf.create (wreck) - local c = { - conf = kvs_input_config (wreck), - wreck = wreck, - maxid = wreck.kvsdir.ntasks - 1, - basedir = tostring (wreck.kvsdir), - inputsdir = tostring (wreck.kvsdir) .. ".input.files", - inputs = {}, - } - -- Add default input config - if not c.conf then - c.conf = { { src = "stdin", dst = "*" } } - end - - c.tasksleft, err = taskset.new ('0-'..c.maxid) - - -- Add stdin entry to inputs - c.inputs[1] = { filename = "stdin", - kzpath = c.inputsdir .. ".stdin" } - return setmetatable (c, inputconf) -end - -function inputconf:set (taskid) - if self.tasksleft[taskid] then - self.tasksleft:clear(taskid) - return false - end - return true -end - -function inputconf:add_source (path) - local i = { filename = path } - local f = self.wreck.flux - - -- First open file for input: - local fp, err = io.open (path, "r") - if not fp then return nil, err end - i.fd = posix.fileno (fp) - - -- now open kz stream for writing: - local basedir = self.inputsdir .. "." .. #self.inputs - f:kvs_put (basedir..".filename", i.filename) - i.kzpath = basedir .. ".kz" - local kz, err = f:kz_open (i.kzpath, "w") - if not kz then - io.close (fp) - return nil, err - end - i.kz = kz - table.insert (self.inputs, i) - return i -end - -function inputconf:input (path) - for _,t in pairs (self.inputs) do - if t.filename == path then - return t - end - end - return self:add_source (path) -end - -function inputconf:render (template, taskid) - local lustache = require 'flux.lustache' - local view = { - taskid = taskid - } - setmetatable (view, { __index = self.wreck }) - return lustache:render (template, view) -end - - -function inputconf:task_input (template, i) - return self:input (self:render (template, i)) -end - -function inputconf:dst_to_list (dst) - if dst == "" or dst == "*" or dst == "all" then - return self.tasksleft - end - return taskset.new (dst) -end - -function inputconf:task_stdin (taskid) - return self.basedir .. "." .. taskid .. ".stdin" -end - --- Link stdin for task id list ids to "path" -function inputconf:link (ids, path) - local f = self.wreck.flux - local taskids = self:dst_to_list (ids) - if taskids:len() == 0 then return true end - for i in taskids:iterator () do - if not self:set (i) then - local input, err = self:task_input (path, i) - if not input then return nil, err end - f:kvs_symlink (self:task_stdin (i), input.kzpath) - end - end - return true -end - -function inputconf:process_config () - for _, input in pairs (self.conf) do - local rc, err = self:link (input.dst, input.src) - if not rc then return nil, err end - end - return true -end -function input_start (f, input) - f:iowatcher { - fd = input.fd, - handler = function (iow, r) - if r.data then input.kz:write (r.data) end - if r.eof then input.kz:close () end - end - } -end - -function inputconf:start () - local f = self.wreck.flux - for path, input in pairs (self.inputs) do - if input.kz and input.fd then - input_start (f, input) - end - end -end -function rexecd_init () - if wreck.nodeid ~= 0 then return end - local f = wreck.flux - local dir = tostring (wreck.kvsdir) - - local cfg = inputconf.create (wreck) - local rc, err = cfg:process_config () - if not rc then - wreck:die ("Error: input: %s", err) - return - end - cfg:start () -end diff --git a/src/modules/wreck/lua.d/intel_mpi.lua b/src/modules/wreck/lua.d/intel_mpi.lua deleted file mode 100644 index c55bc8097f54..000000000000 --- a/src/modules/wreck/lua.d/intel_mpi.lua +++ /dev/null @@ -1,24 +0,0 @@ --- Set environment specific to Intel MPI --- --- Intel PMI is an MPICH derivative that bootstraps with the PMI-1 wire --- protocol, or if I_MPI_PMI_LIBRARY is set, a PMI library. --- --- If the library is set, override it so it points to Flux's. --- If the library is unset, do nothing. --- --- (N.B. We could just unconditionally unset it, but that would prevent the --- user from setting it in order to enable client side PMI tracing in the --- Flux PMI library, enabled by setting FLUX_PMI_DEBUG=1) - - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local libpmi = f:getattr ('conf.pmi_library_path') - - if env['I_MPI_PMI_LIBRARY'] ~= nil then - env['I_MPI_PMI_LIBRARY'] = libpmi - end -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/mvapich.lua b/src/modules/wreck/lua.d/mvapich.lua deleted file mode 100644 index 192c9021b795..000000000000 --- a/src/modules/wreck/lua.d/mvapich.lua +++ /dev/null @@ -1,31 +0,0 @@ --- Set environment specific to mvapich - - -local dirname = require 'flux.posix'.dirname - --- XXX: MVAPICH2 at least requires MPIRUN_RSH_LAUNCH to be set --- in the environment or PMI doesn't work (for unknown reason) --- - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local libpmi = f:getattr ('conf.pmi_library_path') - local ldpath = dirname (libpmi) - - if (env['LD_LIBRARY_PATH'] ~= nil) then - ldpath = ldpath..':'..env['LD_LIBRARY_PATH'] - end - - env['LD_LIBRARY_PATH'] = ldpath - env['MPIRUN_NTASKS'] = wreck.kvsdir.ntasks - env['MPIRUN_RSH_LAUNCH'] = 1 -end - - -function rexecd_task_init () - local env = wreck.environ - env['MPIRUN_RANK'] = wreck.globalid -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/openmpi.lua b/src/modules/wreck/lua.d/openmpi.lua deleted file mode 100644 index 7a5a4c411063..000000000000 --- a/src/modules/wreck/lua.d/openmpi.lua +++ /dev/null @@ -1,16 +0,0 @@ --- Set environment specific to openmpi --- - -local dirname = require 'flux.posix'.dirname - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local rundir = f:getattr ('broker.rundir') - - -- Avoid shared memory segment name collisions - -- when flux instance runs >1 broker per node. - env['OMPI_MCA_orte_tmpdir_base'] = rundir -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/output.lua b/src/modules/wreck/lua.d/output.lua deleted file mode 100644 index 25bb4e71943d..000000000000 --- a/src/modules/wreck/lua.d/output.lua +++ /dev/null @@ -1,126 +0,0 @@ --- --- Optionally write job output to file(s) if so configured in either --- lwj.output or lwj..output --- --- Output filenames are rendered mustache templates --- --- -local ioplex = require 'wreck.io' -local flux = require 'flux' - -local function kvs_output_config (wreck) - local o = wreck.kvsdir ["output"] - if not o then - o = wreck.flux:kvsdir ("lwj.output") - end - return o -end - -local function render (template, wreck, taskid) - local lustache = require 'flux.lustache' - local view = { - cmd = wreck.argv[0]:match("([^/]+)$"), - taskid = taskid - } - setmetatable (view, { __index = wreck }) - return lustache:render (template, view) -end - -local function openstream (wreck, taskio, taskid, stream, template) - local path = render (template, wreck, taskid) - if not path then - wreck:die ("output: Failed to render template '"..template.."'") - return - end - -- Make any kvs:// output relative to the current kvsdir - local key = path:match ("kvs://(.*)$") - if key then - path = "kvs://"..tostring (wreck.kvsdir) .."."..key - end - taskio:redirect (taskid, stream, path) - return path -end - -local function log (fmt, ...) - wreck:log_msg (fmt, ...) -end - -local function log_err (fmt, ...) - wreck:log_error (fmt, ...) -end - --- Read `ioservice` entry for this job, and map any stream with --- rank == FLUX_NODEID_RANK to this rank. --- -local function fetch_ioservices (wreck) - local ioservice,err = wreck.kvsdir.ioservice - if not ioservice then return nil end - local rank = wreck.flux.rank - - -- Only streams with rank == -1 are handled by - -- this plugin. Delete other entries and remap FLUX_NODEID_ANY - -- to this rank: - for s,v in pairs (ioservice) do - if v and v.rank == -1 then - ioservice[s].rank = rank - else - ioservice[s] = nil - end - end - return ioservice -end - -function rexecd_init () - if wreck.nodeid ~= 0 then return end - - local output = kvs_output_config (wreck) - if not output or not output.files then return end - - local template = output.files.stdout - local stderr_template = output.files.stderr - if not template and not stderr_template then return end - - local ntasks = wreck.kvsdir.ntasks - local ioservices = fetch_ioservices (wreck) - - taskio, err = ioplex.create { - flux = wreck.flux, - jobid = wreck.id, - labelio = output.labelio and output.labelio ~= false, - log_err = log_err, - nokz = wreck.kvsdir.options.nokz, - ioservices = ioservices - } - if not taskio then - wreck:log_error ("Error: %s", err) - return - end - - ioplex:enable_debug (log) - - for i = 0, ntasks - 1 do - if template then - openstream (wreck, taskio, i, "stdout", template) - end - if stderr_template then - openstream (wreck, taskio, i, "stderr", stderr_template) - elseif template then - taskio:dup (i, "stderr", "stdout") - end - end - taskio:start () -end - -function rexecd_exit () - if wreck.nodeid ~= 0 or not taskio then return end - while not taskio:complete() do - local rc, err = wreck.flux:reactor ("once") - if not rc then - log_err ("rexecd_exit: reactor once failed: %s", err or "No error") - return - end - end - wreck:log_msg ("File io complete") -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/pmi-mapping.lua b/src/modules/wreck/lua.d/pmi-mapping.lua deleted file mode 100644 index acf3de6ad738..000000000000 --- a/src/modules/wreck/lua.d/pmi-mapping.lua +++ /dev/null @@ -1,38 +0,0 @@ --- Set pmi.PMI_process_mapping - - --- compute blocks list as a Lua table from tasks_per_node and nnodes: -local function compute_blocks (cpn, nnodes) - local blocks = {} - local last = nil - for i = 0, nnodes - 1 do - local count = cpn [i] - if last and cpn[i] == last.tasks then - last.count = last.count + 1 - else - last = { start = i, count = 1, tasks = cpn [i] } - table.insert (blocks, last) - end - end - return blocks -end - --- return 'standard' PMI_process_mapping vector as a string -local function blocks_to_pmi_mapping (blocks) - if not blocks then return " " end - local s = "(vector" - for _,b in pairs (blocks) do - s = s .. string.format (",(%d,%d,%d)", b.start, b.count, b.tasks) - end - s = s .. ")" - return (s) -end - -function rexecd_init () - if (wreck.nodeid ~= 0) then return end - - local blocks = compute_blocks (wreck.tasks_per_node, wreck.nnodes) - local mapping = blocks_to_pmi_mapping (blocks) - wreck.kvsdir ["pmi.PMI_process_mapping"] = mapping -end - diff --git a/src/modules/wreck/lua.d/spectrum.lua b/src/modules/wreck/lua.d/spectrum.lua deleted file mode 100644 index 6985a4a1e984..000000000000 --- a/src/modules/wreck/lua.d/spectrum.lua +++ /dev/null @@ -1,49 +0,0 @@ --- Set environment specific to spectrum_mpi (derived from openmpi) --- - -if wreck:getopt ("mpi") ~= "spectrum" then return end - -local posix = require 'posix' - -function prepend_path (env_var, path) - local env = wreck.environ - if env[env_var] == nil then - suffix = '' - else - suffix = ':'..env[env_var] - end - env[env_var] = path..suffix -end - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local rundir = f:getattr ('broker.rundir') - - -- Avoid shared memory segment name collisions - -- when flux instance runs >1 broker per node. - env['OMPI_MCA_orte_tmpdir_base'] = rundir - - -- Assumes the installation paths of Spectrum MPI on LLNL's Sierra - env['OMPI_MCA_osc'] = "pt2pt" - env['OMPI_MCA_pml'] = "yalla" - env['OMPI_MCA_btl'] = "self" - env['MPI_ROOT'] = "/opt/ibm/spectrum_mpi" - env['OPAL_LIBDIR'] = "/opt/ibm/spectrum_mpi/lib" - env['OMPI_MCA_coll_hcoll_enable'] = '0' - - env['PMIX_SERVER_URI'] = nil - env['PMIX_SERVER_URI2'] = nil - - -- Help find libcollectives.so - prepend_path ('LD_LIBRARY_PATH', '/opt/ibm/spectrum_mpi/lib/pami_port') - prepend_path ('LD_PRELOAD', '/opt/ibm/spectrum_mpi/lib/libpami_cudahook.so') -end - -function rexecd_task_init () - -- Approximately `ulimit -Ss 10240` - -- Used to silence IBM MCM warnings - posix.setrlimit ("stack", 10485760) -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/timeout.lua b/src/modules/wreck/lua.d/timeout.lua deleted file mode 100644 index 3e61bed34a30..000000000000 --- a/src/modules/wreck/lua.d/timeout.lua +++ /dev/null @@ -1,61 +0,0 @@ --- --- Register timer on nodeid 0 if kvs `walltime` is set for this job. --- Kill job on timeout if reached. --- -local posix = require 'flux.posix' - -local function signal_to_number (s) - if not s then return nil end - local ret = tonumber (s) - if ret then return ret end - return posix [s] -end - -local function timeout_signal (f, wreck) - -- If 'lwj.walltime-signal' set then return this signal number, - -- otherwise if 'lwj.walltime-signal' set use this number, - -- otherwise return default signal 'SIGALRM' - -- - local s = signal_to_number (wreck.kvsdir ['walltime-signal']) - if s then return s end - local s = signal_to_number (f:kvsdir() ["lwj.walltime-signal"]) - if s then return s end - return posix.SIGALRM -end - -local function start_timer (f, wreck, id, walltime) - local to, err = f:timer { - timeout = walltime*1000, - oneshot = true, - handler = function (f, to) - local signum = timeout_signal (f, wreck) - wreck:log_error ("Timeout expired! Killing job with signal %d", signum) - local rc,err = f:sendevent ({signal = signum}, "wreck.%d.kill", id) - if not rc then - wreck:log_msg ("Failed to send kill event: %s", err) - end - end - } -end - -function rexecd_init () - if wreck.nodeid ~= 0 then return end - - local walltime = tonumber (wreck.kvsdir.walltime) - if not walltime or walltime <= 0 then return end - - local id = wreck.id - local f = wreck.flux - -- start timer when state is running - local k, err = f:kvswatcher { - key = tostring (wreck.kvsdir) .. ".state", - handler = function (k, result) - if result == "running" then - wreck:log_msg ("detected job state = running, starting timer for %ds", walltime) - k:remove() - start_timer (f, wreck, id, walltime) - end - end - } -end - diff --git a/src/modules/wreck/luastack.c b/src/modules/wreck/luastack.c deleted file mode 100644 index d8a76a0cad96..000000000000 --- a/src/modules/wreck/luastack.c +++ /dev/null @@ -1,455 +0,0 @@ -/************************************************************\ - * Copyright 2014 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "list.h" -#include "luastack.h" - -/**************************************************************************** - * Datatypes - ****************************************************************************/ - -#define LUA_SCRIPT_FILE 1 -#define LUA_SCRIPT_STRING 2 - -struct lua_script { - int type; /* Type of loaded script, a file or a raw string */ - char * data; /* Data: File path or string */ - char * label;/* Filename or name of this script (for errors) */ - - lua_stack_t st; /* Pointer back to lua_stack in which we're loaded */ - lua_State * L; /* Copy of Lua state */ - int env_ref; /* Reference for _ENV table (5.1 setfenv table) */ -}; - -struct lua_script_stack { - lua_State *L; /* Global lua state */ - l_err_f errf; - List script_list; /* List of scripts in this stack */ -}; - - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static int lua_script_is_file (lua_script_t s) -{ - return (s->type == LUA_SCRIPT_FILE); -} - -static int lua_script_is_string (lua_script_t s) -{ - return (s->type == LUA_SCRIPT_STRING); -} - -static void lua_script_destroy (struct lua_script *s) -{ - if (s == NULL) - return; - if (s->L && s->st) { - luaL_unref (s->L, LUA_REGISTRYINDEX, s->env_ref); - /* Only call lua_close() on global/main lua state */ - s->st = NULL; - s->L = NULL; - } - s->type = 0; - free (s->data); - free (s->label); - free (s); -} - - -lua_script_t lua_script_create (lua_stack_t st, int type, const char *data) -{ - struct lua_script *script; - struct lua_State *L; - - if (!st || !st->L) - return (NULL); - - if (type != LUA_SCRIPT_FILE && type != LUA_SCRIPT_STRING) { - (*st->errf) ("lua_script_create: Invalid type!"); - return (NULL); - } - - if ((script = malloc (sizeof (*script))) == NULL) - return (NULL); - - memset (script, 0, sizeof (*script)); - - script->type = type; - if (!(script->data = strdup (data))) { - lua_script_destroy (script); - return (NULL); - } - - if (type == LUA_SCRIPT_FILE && - !(script->label = strdup (basename (script->data)))) { - lua_script_destroy (script); - return (NULL); - } - - L = st->L; - script->st = st; - script->L = L; - - /* New globals table/_ENV for this chunk */ - lua_newtable (script->L); - - /* metatable for table on top of stack */ - lua_newtable (script->L); - - /* - * Now set metatable->__index to point to the real globals - * table. This way Lua will check the root global table - * for any nonexistent items in the current chunk's environment - * table. - */ - lua_pushstring (script->L, "__index"); - lua_getglobal (script->L, "_G"); - lua_settable (script->L, -3); - - /* Now set metatable for the new globals table */ - lua_setmetatable (script->L, -2); - - /* Save reference to this table, which will be used as _ENV for loaded chunk */ - script->env_ref = luaL_ref (script->L, LUA_REGISTRYINDEX); - return (script); -} - -static inline void print_lua_script_error (lua_stack_t st, lua_script_t s) -{ - (*st->errf) ("%s: %s\n", s->label, lua_tostring (s->L, -1)); -} - -static int lua_script_compile (lua_stack_t st, lua_script_t s) -{ - /* - * First load Lua script (via loadfile or loadbuffer) - */ - if (lua_script_is_file (s) && luaL_loadfile (s->L, s->data)) { - (*st->errf) ("%s: Script failed to load.\n", s->data); - return (-1); - } - else if (lua_script_is_string (s) && - luaL_loadbuffer (s->L, s->data, strlen (s->data), s->label)) { - (*st->errf) ("%s: Failed to load script.\n", s->data); - return (-1); - } - - /* Get the environment/globals table for this script from - * the registry and set it as globals table for this chunk - */ - lua_rawgeti (s->L, LUA_REGISTRYINDEX, s->env_ref); -#if LUA_VERSION_NUM >= 502 - /* 5.2 and greater: set table as first upvalue: i.e. _ENV */ - lua_setupvalue (s->L, -2, 1); -#else - /* 5.0, 5.1: Set table as function environment for the chunk */ - lua_setfenv (s->L, -2); -#endif - - /* - * Now compile the loaded script: - */ - if (lua_pcall (s->L, 0, 0, 0)) { - print_lua_script_error (st, s); - return (-1); - } - - return (0); -} - -static int ef (const char *p, int eerrno) -{ - /* FIXME */ - //fprintf (stderr, "glob: %s: %s\n", p, strerror (eerrno)); - return (-1); -} - -int lua_script_list_append (lua_stack_t st, const char *pattern) -{ - glob_t gl; - size_t i; - int rc; - int type = LUA_SCRIPT_FILE; - - if (pattern == NULL || st == NULL || st->script_list == NULL) - return (-1); - - rc = glob (pattern, GLOB_ERR, ef, &gl); - switch (rc) { - case 0: - for (i = 0; i < gl.gl_pathc; i++) { - struct lua_script *s; - if (!(s = lua_script_create (st, type, gl.gl_pathv[i])) || - (lua_script_compile (st, s) < 0)) { - (*st->errf) ("%s: Failed. Skipping.\n", gl.gl_pathv[i]); - lua_script_destroy (s); - continue; - } - list_append (st->script_list, s); - } - break; - case GLOB_NOMATCH: - break; - case GLOB_NOSPACE: - (*st->errf) ("glob: Out of memory\n"); - break; - case GLOB_ABORTED: - (*st->errf) ("Cannot read %s: %s\n", pattern, strerror (errno)); - break; - default: - (*st->errf) ("Unknown glob rc = %d\n", rc); - break; - } - - globfree (&gl); - return (0); -} - -static int lua_script_rc (lua_State *L) -{ - return (lua_isnumber (L, -1) ? lua_tonumber (L, -1) : 0); -} - -static void verr (const char *fmt, ...) -{ - va_list ap; - va_start (ap, fmt); - vfprintf (stderr, fmt, ap); - va_end (ap); -} - -void lua_stack_destroy (lua_stack_t st) -{ - if (st == NULL) - return; - if (st->script_list) - list_destroy (st->script_list); - if (st->L) - lua_close (st->L); - free (st); -} - -lua_stack_t lua_stack_create (void) -{ - lua_stack_t s = malloc (sizeof (*s)); - - if (s == NULL) - return (NULL); - - if (!(s->script_list = list_create ((ListDelF) lua_script_destroy))) { - free (s); - return (NULL); - } - - s->L = luaL_newstate (); - s->errf = &verr; - - luaL_openlibs(s->L); - - return (s); -} - -int lua_stack_append_file (lua_stack_t st, const char *pattern) -{ - if (st == NULL || st->script_list == NULL) - return (-1); - - if (lua_script_list_append (st, pattern) < 0) - return (-1); - - return (0); -} - -int lua_stack_append_script (lua_stack_t st, const char *script, - const char *label) -{ - int type = LUA_SCRIPT_STRING; - lua_script_t s = lua_script_create (st, type, script); - if (s == NULL) - return (-1); - - s->label = label ? strdup (label) : strdup ("