diff --git a/README.md b/README.md index 917fa3f07eef..966bcd819303 100644 --- a/README.md +++ b/README.md @@ -139,8 +139,6 @@ Common commands from flux-core: start bootstrap a local Flux instance submit submit job requests to a scheduler user Flux user database client - wreck Flux wreck convenience utilities - wreckrun Flux utility for remote execution ``` Most of these have UNIX manual pages as `flux-(1)`, diff --git a/configure.ac b/configure.ac index 00f6ce673779..536eed384a52 100644 --- a/configure.ac +++ b/configure.ac @@ -388,7 +388,6 @@ AC_CONFIG_FILES( \ src/common/libflux/version.h \ src/common/libkvs/Makefile \ src/common/libkz/Makefile \ - src/common/libjsc/Makefile \ src/common/libjob/Makefile \ src/common/libzio/Makefile \ src/common/libsubprocess/Makefile \ @@ -416,7 +415,6 @@ AC_CONFIG_FILES( \ src/modules/kvs-watch/Makefile \ src/modules/content-sqlite/Makefile \ src/modules/barrier/Makefile \ - src/modules/wreck/Makefile \ src/modules/cron/Makefile \ src/modules/aggregator/Makefile \ src/modules/pymod/Makefile \ diff --git a/doc/man1/Makefile.am b/doc/man1/Makefile.am index 7df0973c7fcf..b2a8d7f60fb9 100644 --- a/doc/man1/Makefile.am +++ b/doc/man1/Makefile.am @@ -8,9 +8,6 @@ MAN1_FILES_PRIMARY = \ flux-keygen.1 \ flux-logger.1 \ flux-ping.1 \ - flux-submit.1 \ - flux-wreckrun.1 \ - flux-wreck.1 \ flux-start.1 \ flux-module.1 \ flux-exec.1 \ @@ -60,8 +57,6 @@ stderr_devnull_0 = 2>/dev/null EXTRA_DIST = \ $(ADOC_FILES) \ COPYRIGHT.adoc \ - NODESET.adoc \ - wreck-options.adoc \ - wreck-extra-options.adoc + NODESET.adoc CLEANFILES = $(MAN1_FILES) $(XML_FILES) diff --git a/doc/man1/flux-start.adoc b/doc/man1/flux-start.adoc index c1d5e1c38d1a..aff655b74bcc 100644 --- a/doc/man1/flux-start.adoc +++ b/doc/man1/flux-start.adoc @@ -83,12 +83,6 @@ shell as the initial program: srun --pty -N8 flux start -Launch an 8-way Flux instance under Flux, with /bin/true as -the initial program: - - flux wreckrun -N8 flux start /bin/true - - AUTHOR ------ This page is maintained by the Flux community. diff --git a/doc/man1/flux-submit.adoc b/doc/man1/flux-submit.adoc deleted file mode 100644 index ea86afcd6a90..000000000000 --- a/doc/man1/flux-submit.adoc +++ /dev/null @@ -1,55 +0,0 @@ -// flux-help-description: submit job requests to a scheduler -FLUX-SUBMIT(1) -============== -:doctype: manpage - - -NAME ----- -flux-submit - Flux utility for submitting job requests to a scheduler - - -SYNOPSIS --------- -[verse] -'flux submit' [-n ] [-N ] [-t ] - [-o|--options='OPTIONS'] - [-O|--output='FILENAME'] [-E|--error='FILENAME'] - [-i|--input='HOW'] ['COMMANDS'...] - - -DESCRIPTION ------------ - -'flux submit' is the front-end command for submitting jobs to -a Flux comms session for eventual execution by a Flux scheduler. -Once a job has been scheduled for execution, it will be executed -using the WRECK remote execution prototype. - -See flux-wreckrun(1) for more information about execution -method. - -OPTIONS -------- - -include::wreck-options.adoc[] - -include::wreck-extra-options.adoc[] - -AUTHOR ------- -This page is maintained by the Flux community. - - -RESOURCES ---------- -Github: - - -COPYRIGHT ---------- -include::COPYRIGHT.adoc[] - -SEE ALSO --------- -flux-wreckrun(1), flux-wreck(1) diff --git a/doc/man1/flux-wreck.adoc b/doc/man1/flux-wreck.adoc deleted file mode 100644 index 50709c526b9b..000000000000 --- a/doc/man1/flux-wreck.adoc +++ /dev/null @@ -1,177 +0,0 @@ -// flux-help-description: Flux wreck convenience utilities -FLUX-WRECK(1) -============= -:doctype: manpage - - -NAME ----- -flux-wreck - Flux wreck facility convenience utility - - -SYNOPSIS --------- -*flux* *wreck* 'COMMAND' ['OPTIONS'] - - -DESCRIPTION ------------ -'flux wreck' is a convenience utility that operates on jobs in the -WRECK remote execution prototype. It can be used to gather information -about running and completed jobs directly from the Flux kvs. - -flux-wreck(1) operates via a series of 'COMMANDS', described below: - -COMMANDS --------- -*help* 'cmd':: -Print help. If 'cmd' is provided, print help for that sub-command. - -*ls* [-n, --max=COUNT] [JOBIDS...]:: -Display a list of wreck jobs currently in kvs, along with their current -states. If '-n, --max' option is provided, then display at most 'COUNT' -jobs (default: 25). If an optional list of 'JOBIDS' is provided on the -command line, then display only those jobs. - -*uri* [-n, --max=COUNT] [-b, --bare] [JOBIDS...]:: -Display a list of wreck jobs currently in kvs, with abbreviated job info, -and a URI that can be used to contact the rank 0 broker if the job is a -Flux instance. The field is blank for other types of jobs. -If '-n, --max' option is provided, then display at most 'COUNT' -jobs (default: 25). If '-b, --bare' option is provided, display only the -URI for a single job so that it can be parsed by scripts. If an optional -list of 'JOBIDS' is provided on the command line, then display only those jobs. - -*attach* [--status] [--label-io] 'jobid':: -Attach to output of a running or completed job. If input was not previously -connected, also attach to stdin. With '--status', also report job status -after dumping output. With '--label-io' label lines of output with taskid. - -*status* 'jobid':: -Return the status of running or completed job 'jobid'. Prints the current -job status, and if exited, the exit codes of all tasks. This command will -exit with the highest exit code of all tasks. - -*cancel* 'jobid':: -Make a request to the scheduler to cancel a pending job 'jobid'. Only supported -if a job scheduler module is loaded. - -*kill* [--signal=N] 'jobid':: -Send SIGTERM to running job 'jobid'. If '--signal' is used, send signal 'N' -where 'N' may be a signal number or name such as 'SIGKILL'. - -*setenv* [VAR=VALUE|all]:: -Export environment variables to all future jobs. The 'flux wreck setenv', -'getenv' and 'unsetenv' commands control the global job environment which -is applied to all jobs run by the wreck system. With 'flux wreck setenv all' -the entire environment is exported to the global job environment, inherited -by all future jobs. This may help reduce the size of job environment records -stored in the KVS if there are only small environment differences from -run to run. 'flux wreck setenv VAR=VALUE' may also be used to export a -single environment variable to all jobs. - -*getenv* VAR:: -Get the value of an environment variable set in the global job environment. - -*unsetenv* VAR:: -Unset environment variable 'VAR' in the global job environment. - -*setopt* 'option[=value]':: -Set global job option 'option' with optional value 'value' (the default value -is 'true'). All future jobs will inherit global job options as if they -were run with '-o option=value'. - -*getopt* '[option]':: -Display global job options. - -*timing* [-n, --max=COUNT] [JOBIDS...]:: -List timing information for lwj entries in Flux KVS. if '-n, --max' option is -provided, then display at most 'COUNT' jobs (default: 25). If an optional list -of 'JOBIDS' is provided on the command line, then display timing information -for only those jobs. This command lists four specific intervals in the wreck -job lifecycle: - 'STARTING'::: Time from LWJ entry creation until the job reaches the - "starting" state. The "starting" state is when the first - rexec daemon receives the request to launch tasks. - 'RUNNING'::: Time from "starting" to "running". The running state is - reached when all rexec daemons have forked and executed - all tasks. - 'COMPLETE'::: Time from "running" to "complete". This is the nominal time - that tasks of the job actually ran. - 'TOTAL'::: This is the total time from job creation to the "completed" state. - - -*kvs-path* 'jobid':: -Display the path in the Flux KVS to the specified 'jobid'. - -*last-jobid* [--kvs-path]:: -Return the current value of the 'lwj' sequence number, that is the -last jobid allocated to a submitted wreck job. With '--kvs-path', -report the path to job in kvs instead of jobid. Other jobs may be -started between the time the id is obtained and when it is displayed -in this command, so 'last-jobid' should not be relied upon during real -workloads. - -*purge* [OPTIONS...]:: -Purge old job entries from Flux KVS. Without any options, the 'purge' -subcommand will print the current number of entries in the 'lwj.' -kvs directory. Otherwise, 'flux wreck purge' walks the list of all -completed jobs in the Flux KVS and selects the oldest (in terms of -completion time) as candidates for removal. The number of jobs selected -is dependent on the options below: - ---verbose::: --v::: - Increase verbosity. - ---target-size='COUNT'::: --t 'COUNT'::: - Set the target number of job entries to keep to 'COUNT'. The number - of IDs to remove will be calculated based on target size, number of - current job entries, and any maximum set on command line. - ---remove::: --R::: - Perform an unlink of all selected job entries in KVS. This option is - only valid if supplied with '--target-size'. - ---max=N::: --m N::: - Set the maximum number of entries to remove on any one run. - - -*dumplog* [JOBID]:: -Dump the error log from the wreck job 'JOBID'. - -*exclude* [NODENAME]:: -Exclude a node from running future jobs. By default, jobs currently running -on the node and its resources are not killed. - --k, --kill::: - Kill all jobs to which the node is allocated. - -*include* [NODENAME]:: -Include a previously excluded node from scheduling. - -*sched-params* set 'item=val':: -*sched-params* get 'item':: -Set and get scheduler parameters at runtime. - -AUTHOR ------- -This page is maintained by the Flux community. - - -RESOURCES ---------- -Github: - - -COPYRIGHT ---------- -include::COPYRIGHT.adoc[] - - -SEE ALSO --------- -flux-wreckrun(1) diff --git a/doc/man1/flux-wreckrun.adoc b/doc/man1/flux-wreckrun.adoc deleted file mode 100644 index f3568cffb95f..000000000000 --- a/doc/man1/flux-wreckrun.adoc +++ /dev/null @@ -1,250 +0,0 @@ -// flux-help-include: yes -FLUX-WRECKRUN(1) -================ -:doctype: manpage - - -NAME ----- -flux-wreckrun - Flux utility for remote execution - - -SYNOPSIS --------- -[verse] -'flux wreckrun' [-n ] [-N ] [-t ] - [-l|--label-io] [-d|--detach] [-o|--options='OPTIONS'] - [-O|--output='FILENAME'] [-E|--error='FILENAME'] - [-i|--input='HOW'] [-I, --immediate] ['COMMANDS'...] - - -DESCRIPTION ------------ - -'flux wreckrun' is the front-end command for launching tasks within -a Flux comms session using the WRECK remote execution prototype. - -WRECK (Worthless Remote Execution using CMB KVS) is a prototype -implementation of remote execution for Flux which demonstrates -the usage of the Flux key-value store as the vehicle for transmitting -and storing information about lightweight jobs (LWJs). The basic -mode of operation is described in xref:wreck-operation[]. - -OPTIONS -------- - -include::wreck-options.adoc[] - ---detach:: --d:: - Detach immediately after issuing a run request and print new jobid - to stdout. - ---wait-until='state':: --w 'state':: - Ignore all stdio, and instead just wait until the job reaches - a state of 'state' (i.e. "starting", "running", or "complete") and - exit immediately. - ---immediate:: --I:: - Bypass scheduler and run job immediately. - - -include::wreck-extra-options.adoc[] - -OPERATION ----------- -[[wreck-operation]] - -Briefly, the method of operation for remote execution via WRECK -consists of the following steps: - -. The front-end command ('wreckrun') allocates a new job directory - entry in the kvs under the 'lwj' key. This operation is performed as - a 'job.create' request, which results in a new directory in the kvs, - such as 'lwj.'. The full directory path is dependent on - settings but can be determined via "flux wreck last-jobid -p". - Initially, a job state is 'reserved' and stored in the 'state' - directory, such as 'lwj..state'. -. The front-end command now fills in requisite and optional job - information under the 'lwj' jobid directory such as the command-line - to run, how many tasks to run on each rank, the job environment, - etc. -. The front-end command then issues a 'wrexec.run' event, which - is handled by the 'wrexec' module. The 'wrexec' module spawns - a daemon to handle the request for the jobid in question. The - 'wrexecd' daemon pulls information for the job in question - directly from the local kvs cache and spawns required tasks. -. Once the first task is started by rank 0, the job state is updated - to 'starting'. -. After starting tasks, all 'wrexecd' daemons synchronize on a - barrier, and the job state is updated to 'running'. -. As the tasks run, stdio is either stored in the KVS or sent directly - to a per-job service as messages (when '-o nokz' job option is set), - and output is optionally displayed to user or redirected as requested - by command line options. -. As tasks exit, their exit status is recorded in the kvs. -. After all tasks have exited, 'wrexecd' daemons synchronize again - and rank 0 updates the job state to 'completed'. - -This simple scheme offers a scalable and semi-flexible method -for launching tasks within a Flux comms session. In conjunction -with the Flux PMI library, even MPI applications may be launched -at scale via 'flux wreckrun'. See xref:wreckrun-examples[] for -examples. - -OUTPUT ENVIRONMENT VARIABLES ----------------------------- -[[wreckrun-output-environment-variables]] - -The following environment variables will be set in the environment of -tasks executed via 'flux wreckrun'. - -*FLUX_JOB_ID*:: -The job ID of the current job - -*FLUX_JOB_NNODES*:: -Number of nodes used in the current job - -*FLUX_NODE_ID*:: -The relative node ID of the current node - -*FLUX_JOB_SIZE*:: -The total number of tasks to be run in the current job - -*FLUX_LOCAL_RANKS*:: -Comma separated list of task ranks run on the current node - -*FLUX_TASK_RANK*:: -The relative task rank of the current task - -*FLUX_TASK_LOCAL_ID*:: -The relative task rank of the current task on the current node - - -EXAMPLES --------- -[[wreckrun-examples]] - -Once a Flux comms session is running, ensure the required modules -for 'wreck' are loaded: - ------------------------------- -$ flux module list | egrep 'resource|job|wrexec' -resource-hwloc 44673 498403A idle 0 -wrexec 42052 2CA4CAT idle 0 -job 28959 60947AA idle 0 -$ ------------------------------- - -To ensure wrexec is working properly, run hostname across -4 nodes of the Flux comms session: - ------------------------------- -$ flux wreckrun -l -N4 hostname -wreckrun: 0.000s: Sending LWJ request for 1 tasks (cmdline "hostname") -wreckrun: 0.007s: Registered jobid 1 -wreckrun: Allocating 4 tasks across 4 available nodes.. -wreckrun: tasks per node: node[0-3]: 1 -wreckrun: 0.009s: Sending run event -wreckrun: 0.012s: State = reserved -wreckrun: 0.020s: State = starting -wreckrun: 0.082s: State = running -0: hype349 -2: hype351 -3: hype352 -1: hype350 -wreckrun: 0.140s: State = complete -wreckrun: All tasks completed successfully. ------------------------------- - -'flux wreckrun' uses the Flux kvs for most of its functionality, -and thus information about lightweight jobs (LWJs) that are -executed will be stored in the 'lwj.' directory in the kvs. -To see what kind of information is stored there, use the -'flux kvs' utility: - ------------------------------- -$ flux kvs dir lwj.1 ------------------------------- - -.Running an MPI application with 'flux wreckrun' - -Launching an MPI application with 'flux wreckrun' requires that your -version of MPI supports PMI. In this example, MVAPICH2 is used: - ------------------------------- -$ mpicc -v 2>&1 | head -1 -mpicc for MVAPICH2 version 1.7 -$ mpicc -o mpi_hello mpi_hello.c ------------------------------- - -In order to run using the Flux PMI library, 'LD_LIBRARY_PATH' may need -to be adjusted at runtime. For example, to use Flux's libpmi.so -from the build directory: - ------------------------------- -$ export LD_LIBRARY_PATH=${flux_builddir}/src/lib/libpmi/.libs${LD_LIBRARY_PATH} ------------------------------- - -where '${flux_builddir}' is the path to the root of the flux-core -build directory. - -Now the MPI application should run via 'flux wreckrun': - ------------------------------- -$ flux wreckrun -N4 ./mpi_hello -wreckrun: 0.000s: Sending LWJ request for 1 tasks (cmdline "./mpi_hello") -wreckrun: 0.007s: Registered jobid 5 -wreckrun: Allocating 4 tasks across 4 available nodes.. -wreckrun: tasks per node: node[0-3]: 1 -wreckrun: 0.009s: Sending run event -wreckrun: 0.011s: State = reserved -wreckrun: 0.018s: State = starting -wreckrun: 0.028s: State = running -0: completed first barrier -0: completed MPI_Init in 0.039s. There are 4 tasks -0: completed MPI_Finalize -wreckrun: 2.182s: State = complete -wreckrun: All tasks completed successfully. ------------------------------- - -The use of Flux PMI can be verified by checking the KVS 'pmi' directory -for the LWJ in question: - ------------------------------- -$ flux kvs dir pmi.5 -pmi.5.hostname[2] = "-1463773063" -pmi.5.hostname[0] = "-1463773575" -pmi.5.hostname[1] = "-1463773319" -pmi.5.hostname[3] = "-1463772807" -pmi.5.uuid_2_5 = "dummy-entry" -pmi.5.uuid_0_5 = "8hiWSLCPRzqFYHNubktbCQ==" -pmi.5.uuid_1_5 = "dummy-entry" -pmi.5.uuid_3_5 = "dummy-entry" -pmi.5.pmi_epidkey_2 = "1901059" -pmi.5.pmi_epidkey_0 = "7209475" -pmi.5.pmi_epidkey_1 = "6816259" -pmi.5.pmi_epidkey_3 = "6750723" ------------------------------- - - -AUTHOR ------- -This page is maintained by the Flux community. - - -RESOURCES ---------- -Github: - - -COPYRIGHT ---------- -include::COPYRIGHT.adoc[] - - -SEE ALSO --------- -flux-wreck(1) diff --git a/doc/man1/wreck-extra-options.adoc b/doc/man1/wreck-extra-options.adoc deleted file mode 100644 index a6a5ccb38ef2..000000000000 --- a/doc/man1/wreck-extra-options.adoc +++ /dev/null @@ -1,81 +0,0 @@ -EXTRA OPTIONS -------------- -[[extra-options]] - -Extra options can be supplied on the `flux wreckrun` command via the -'--options' command line option. Multiple options should be separated -via commas. Currently available options include: - -'cpu-affinity[=per-task]':: - Set default CPU affinity of all tasks to the cores allocated - to the job. With 'cpu-affinity=per-task' distribute tasks - across allocated cores using 'hwloc_distrib(3)'. - -'gpubind=[per-task,off]':: - By default, if GPU devices are allocated to a job, the environment - variable CUDA_VISIBLE_DEVICES is set to the list of allocated - GPUs. The 'gpubind' option may be used to disable this behavior - '-o gpubind=off', or can request that the GPUs be distributed - among the local tasks with '-o gpubind=per-task'. - -'mpi':: - Select an MPI type being run if necessary. Some MPI "plugins" - are not enabled by default, and this option may be used to - enable them, e.g. '-o mpi=spectrum'. - -'nokz':: - Do not use KVS based "kz" streams for stdout and stderr streams - of this job. This option will bypass the KVS for job output by - using messages directly sent to a per-job service registered by - either wreckrun or the first wrexecd of the job. - -'kz':: - Force use of KVS based "kz" streams for stdout and stderr streams - for this job. This option may be used in place of '-o nokz=false' - when 'nokz' has been set as the default for all jobs via global - wreck options (i.e. if 'flux wreck getopt nokz' reports true) - -'stdio-commit-on-open':: - Commit to kvs on stdio open in each task. The default is to - delay commit. Without this option, stdio files for jobs will - not appear in KVS on all nodes until the next KVS commit. - The default behavior avoids a kvs commit storm as stdout and - stderr files are opened for every task. - -'stdio-commit-on-close':: - Commit to kvs for each stdio stream as it gets EOF. The default - is to delay the commit. Without this option, stdio files for - jobs will not appear to close until after the next natural - KVS commit. The default behavior avoids a kvs commit storm - as stdout and stderr files are closed from many tasks. This - option may need to be enabled if you need to detect immediately - when a stdout or stderr stream is closed from a task. - -'stdio-delay-commit':: - Disable commit to kvs for each line of output. The default - behavior is to call kvs_commit() after each line of output - from every task. If many tasks are writing multiple lines - of output and it is not necessary to see lines of output - as they are generated, it will speed up job execution to - enable this option. - -'commit-on-task-exit':: - Commit to the kvs for each task exit event. The default behavior - is to write the task exit status to the kvs as each task in - a lightweight job exits, but delay kvs commit. This avoids - a kvs commit storm when many tasks are exiting simultaneously. - This option may need to be enabled if knowlege of exit status or - fact that tasks have exited is required in real time. - -'stop-children-in-exec':: - Start tasks in a STOPPED state waiting for attach from a - debugger. This option is provided for use with parallel - debuggers such as TotalView. - -'no-pmi-server':: - Do not start simple pmi server. This option can be used to - not launch the pmi server if a non-MPI job is being run. - -'trace-pmi-server':: - Log simple pmi server protocol exchange. This option is used - for debugging. diff --git a/doc/man1/wreck-options.adoc b/doc/man1/wreck-options.adoc deleted file mode 100644 index 76c247fef156..000000000000 --- a/doc/man1/wreck-options.adoc +++ /dev/null @@ -1,85 +0,0 @@ - ---ntasks='ntasks':: --n 'ntasks':: - Request to run a total of 'ntasks' tasks. If '--ntasks' is used alone, - i.e. without '--nnodes', then 'flux wreckrun' will attempt to - distribute the 'ntasks' tasks across all nodes of the Flux comms - session. - ---cores-per-task='N':: --c 'N':: - Request N cores be allocated to each task. - ---gpus-per-task='N':: --g 'N':: - Request N GPUs be allocated to each task. - ---nnodes='nnodes':: --N 'nnodes':: - Request to run across 'nnodes' nodes in the current comms session. - ---tasks-per-node='tpn':: --t 'tpn':: - Set the number of tasks to run per node to 'tpn'. The default is - --tasks-per-node=1. This value is overridden by '--ntasks' if used. - ---label-io:: --l:: - Prepend stdout and stderr output lines with the task id to which - the output belongs. - ---output='FILENAME':: --O 'FILENAME':: - Duplicate stdout and stderr from tasks to a file or files. 'FILENAME' - is optionally a mustache template which expands the keys 'id', 'cmd' - and 'taskid'. (e.g. '--output=flux-{{id}}.out') - ---error='FILENAME':: --E 'FILENAME':: - Send stderr to a different location than stdout. - ---input='HOW':: --i 'HOW':: - Indicate how to deal with stdin for tasks. 'HOW' is a list of 'src:dst' - pairs where 'src' is a source 'FILENAME' which may optionally be a - mustache template as in `--output`, or the special term `stdin` to - indicate the stdin from a front end program, and 'dst' is a list of - taskids or `*` or `all` to indicate all tasks. The default is `stdin:*`. - If only one of 'src' or 'dst' is specified, a heuristic is used to - determine whether a list of tasks or an input file was meant. (e.g - `--input=file.in` will use `file.in` as input for all tasks, and - `--input=0` will send stdin to task 0 only. - ---walltime='N[SUFFIX]':: --T 'N[SUFFIX]':: - Set max job walltime to N seconds. Optional suffix may be 's' - for seconds (default), 'm' for minutes, 'h' for hours or 'd' - for days. N may be an arbitrary floating point number, but - will be rounded up to nearest second. - ---skip-env:: --S:: - Skip the export of the local environment to the job. - ---epilog='PATH':: --x 'PATH':: - Execute a script at 'PATH' after all tasks exit but before the - job state is set to to 'complete' (So the job will not complete - and the resources will not be released until the script ends) - ---postscript='PATH':: --p 'PATH':: - Similar to '--epilog', but execute 'PATH' after job state is - complete. - ---options='options,...':: --o 'options,...':: - Apply extended job options to the current execution. Examples of - these options are described in the xref:extra-options[]. - ---name='JOBNAME':: --J 'JOBNAME':: - Set an optional name to 'JOBNAME' for the submitted job. This name - will be saved in the KVS record for the job and displayed in place - of command name in `flux wreck ls` output. - diff --git a/doc/test/spell.en.pws b/doc/test/spell.en.pws index acdebc725f66..796ebb39523e 100644 --- a/doc/test/spell.en.pws +++ b/doc/test/spell.en.pws @@ -96,9 +96,6 @@ runtime src tpn uuid -wreckrun -wrexec -wrexecd ary baz EPGM diff --git a/etc/rc1 b/etc/rc1 index 8b53c82c470c..a7b49f25c0fe 100755 --- a/etc/rc1 +++ b/etc/rc1 @@ -11,14 +11,10 @@ flux module load -r all aggregator flux hwloc reload & pids="$pids $!" -flux module load -r all job flux module load -r 0 cron sync=hb flux module load -r 0 userdb ${FLUX_USERDB_OPTIONS} -# make flux wreckrun/submit -onokz the default (override w/ -o kz): -flux wreck setopt nokz - flux module load -r all job-ingest flux module load -r 0 job-manager diff --git a/etc/rc3 b/etc/rc3 index d98702365e31..189ebf3eb58d 100755 --- a/etc/rc3 +++ b/etc/rc3 @@ -16,13 +16,12 @@ flux module remove -r 0 job-manager flux module remove -r all job-ingest if PERSISTDIR=$(flux getattr persist-directory 2>/dev/null); then - flux wreck ls >${PERSISTDIR}/joblog 2>/dev/null || : + /bin/true; # XXX: nothing to persist? fi flux module remove -r 0 userdb flux module remove -r 0 cron -flux module remove -r all job flux module remove -r all aggregator flux module remove -r all barrier diff --git a/src/bindings/lua/Makefile.am b/src/bindings/lua/Makefile.am index 9a3b248f42c4..2cf346ab6cc3 100644 --- a/src/bindings/lua/Makefile.am +++ b/src/bindings/lua/Makefile.am @@ -10,28 +10,14 @@ AM_CPPFLAGS = \ fluxluadir = $(luadir)/flux fluxometerdir = $(luadir)/fluxometer -wreckluadir = $(luadir)/wreck -lustachedir = $(fluxluadir)/lustache fluxluaexecdir = $(luaexecdir)/flux -dist_lua_SCRIPTS = \ - wreck.lua - nobase_dist_lua_SCRIPTS = \ flux/timer.lua \ flux/alt_getopt.lua \ flux/posix.lua \ flux/base64.lua \ - flux/lustache.lua \ - flux/Subcommander.lua \ - wreck/io.lua - -# Required because install(1) doesn't seem to preserve -# full path when copying to directory: -dist_lustache_SCRIPTS = \ - flux/lustache/context.lua \ - flux/lustache/renderer.lua \ - flux/lustache/scanner.lua + flux/Subcommander.lua nobase_dist_fluxometer_SCRIPTS = \ Test/Builder/NoOutput.lua \ diff --git a/src/bindings/lua/README b/src/bindings/lua/README index 97637ba02a08..cea75e5e2aa6 100644 --- a/src/bindings/lua/README +++ b/src/bindings/lua/README @@ -64,10 +64,6 @@ First, a quick example to display the basics: end end - - For another example, see dlua/wreck.lua in-tree. It is a reimplementation - of wreck/wreckrun in pure Lua. - - ======================= 2. Flux API usage: diff --git a/src/bindings/lua/flux/lustache.lua b/src/bindings/lua/flux/lustache.lua deleted file mode 100644 index b85b1f5580cb..000000000000 --- a/src/bindings/lua/flux/lustache.lua +++ /dev/null @@ -1,27 +0,0 @@ --- lustache: Lua mustache template parsing. --- Copyright 2013 Olivine Labs, LLC --- MIT Licensed. - -local string_gmatch = string.gmatch - -function string.split(str, sep) - local out = {} - for m in string_gmatch(str, "[^"..sep.."]+") do out[#out+1] = m end - return out -end - -local lustache = { - name = "lustache", - version = "1.3-1", - renderer = require("flux.lustache.renderer"):new(), -} - -return setmetatable(lustache, { - __index = function(self, idx) - if self.renderer[idx] then return self.renderer[idx] end - end, - __newindex = function(self, idx, val) - if idx == "partials" then self.renderer.partials = val end - if idx == "tags" then self.renderer.tags = val end - end -}) diff --git a/src/bindings/lua/flux/lustache/context.lua b/src/bindings/lua/flux/lustache/context.lua deleted file mode 100644 index edb0d3573845..000000000000 --- a/src/bindings/lua/flux/lustache/context.lua +++ /dev/null @@ -1,66 +0,0 @@ -local string_find, string_split, tostring, type = - string.find, string.split, tostring, type - -local context = {} - -function context:clear_cache() - self.cache = {} -end - -function context:push(view) - return self:new(view, self) -end - -function context:lookup(name) - local value = self.cache[name] - - if not value then - if name == "." then - value = self.view - else - local context = self - - while context do - if string_find(name, ".") > 0 then - local names = string_split(name, ".") - local i = 0 - - value = context.view - - if(type(value)) == "number" then - value = tostring(value) - end - - while value and i < #names do - i = i + 1 - value = value[names[i]] - end - else - value = context.view[name] - end - - if value then - break - end - - context = context.parent - end - end - - self.cache[name] = value - end - - return value -end - -function context:new(view, parent) - local out = { - view = view, - parent = parent, - cache = {}, - magic = "1235123123", --ohgodwhy - } - return setmetatable(out, { __index = self }) -end - -return context diff --git a/src/bindings/lua/flux/lustache/renderer.lua b/src/bindings/lua/flux/lustache/renderer.lua deleted file mode 100644 index 593f506cd003..000000000000 --- a/src/bindings/lua/flux/lustache/renderer.lua +++ /dev/null @@ -1,374 +0,0 @@ -local Scanner = require "flux.lustache.scanner" -local Context = require "flux.lustache.context" - -local error, ipairs, loadstring, pairs, setmetatable, tostring, type = - error, ipairs, loadstring, pairs, setmetatable, tostring, type -local math_floor, math_max, string_find, string_gsub, string_split, string_sub, table_concat, table_insert, table_remove = - math.floor, math.max, string.find, string.gsub, string.split, string.sub, table.concat, table.insert, table.remove - -local patterns = { - white = "%s*", - space = "%s+", - nonSpace = "%S", - eq = "%s*=", - curly = "%s*}", - tag = "[#\\^/>{&=!]" -} - -local html_escape_characters = { - ["&"] = "&", - ["<"] = "<", - [">"] = ">", - ['"'] = """, - ["'"] = "'", - ["/"] = "/" -} - -local function is_array(array) - if type(array) ~= "table" then return false end - local max, n = 0, 0 - for k, _ in pairs(array) do - if not (type(k) == "number" and k > 0 and math_floor(k) == k) then - return false - end - max = math_max(max, k) - n = n + 1 - end - return n == max -end - --- Low-level function that compiles the given `tokens` into a --- function that accepts two arguments: a Context and a --- Renderer. - -local function compile_tokens(tokens, originalTemplate) - local subs = {} - - local function subrender(i, tokens) - if not subs[i] then - local fn = compile_tokens(tokens, originalTemplate) - subs[i] = function(ctx, rnd) return fn(ctx, rnd) end - end - return subs[i] - end - - local function render(ctx, rnd) - local buf = {} - local token, section - for i, token in ipairs(tokens) do - local t = token.type - buf[#buf+1] = - t == "#" and rnd:_section( - token, ctx, subrender(i, token.tokens), originalTemplate - ) or - t == "^" and rnd:_inverted( - token.value, ctx, subrender(i, token.tokens) - ) or - t == ">" and rnd:_partial(token.value, ctx, originalTemplate) or - (t == "{" or t == "&") and rnd:_name(token.value, ctx, false) or - t == "name" and rnd:_name(token.value, ctx, true) or - t == "text" and token.value or "" - end - return table_concat(buf) - end - return render -end - -local function escape_tags(tags) - - return { - string_gsub(tags[1], "%%", "%%%%").."%s*", - "%s*"..string_gsub(tags[2], "%%", "%%%%"), - } -end - -local function nest_tokens(tokens) - local tree = {} - local collector = tree - local sections = {} - local token, section - - for i,token in ipairs(tokens) do - if token.type == "#" or token.type == "^" then - token.tokens = {} - sections[#sections+1] = token - collector[#collector+1] = token - collector = token.tokens - elseif token.type == "/" then - if #sections == 0 then - error("Unopened section: "..token.value) - end - - -- Make sure there are no open sections when we're done - section = table_remove(sections, #sections) - - if not section.value == token.value then - error("Unclosed section: "..section.value) - end - - section.closingTagIndex = token.startIndex - - if #sections > 0 then - collector = sections[#sections].tokens - else - collector = tree - end - else - collector[#collector+1] = token - end - end - - section = table_remove(sections, #sections) - - if section then - error("Unclosed section: "..section.value) - end - - return tree -end - --- Combines the values of consecutive text tokens in the given `tokens` array --- to a single token. -local function squash_tokens(tokens) - local out, txt = {}, {} - local txtStartIndex, txtEndIndex - for _, v in ipairs(tokens) do - if v.type == "text" then - if #txt == 0 then - txtStartIndex = v.startIndex - end - txt[#txt+1] = v.value - txtEndIndex = v.endIndex - else - if #txt > 0 then - out[#out+1] = { type = "text", value = table_concat(txt), startIndex = txtStartIndex, endIndex = txtEndIndex } - txt = {} - end - out[#out+1] = v - end - end - if #txt > 0 then - out[#out+1] = { type = "text", value = table_concat(txt), startIndex = txtStartIndex, endIndex = txtEndIndex } - end - return out -end - -local function make_context(view) - if not view then return view end - return view.magic == "1235123123" and view or Context:new(view) -end - -local renderer = { } - -function renderer:clear_cache() - self.cache = {} - self.partial_cache = {} -end - -function renderer:compile(tokens, tags, originalTemplate) - tags = tags or self.tags - if type(tokens) == "string" then - tokens = self:parse(tokens, tags) - end - - local fn = compile_tokens(tokens, originalTemplate) - - return function(view) - return fn(make_context(view), self) - end -end - -function renderer:render(template, view, partials) - if type(self) == "string" then - error("Call mustache:render, not mustache.render!") - end - - if partials then - -- remember partial table - -- used for runtime lookup & compile later on - self.partials = partials - end - - if not template then - return "" - end - - local fn = self.cache[template] - - if not fn then - fn = self:compile(template, self.tags, template) - self.cache[template] = fn - end - - return fn(view) -end - -function renderer:_section(token, context, callback, originalTemplate) - local value = context:lookup(token.value) - - if type(value) == "table" then - if is_array(value) then - local buffer = "" - - for i,v in ipairs(value) do - buffer = buffer .. callback(context:push(v), self) - end - - return buffer - end - - return callback(context:push(value), self) - elseif type(value) == "function" then - local section_text = string_sub(originalTemplate, token.endIndex+1, token.closingTagIndex - 1) - - local scoped_render = function(template) - return self:render(template, context) - end - - return value(section_text, scoped_render) or "" - else - if value then - return callback(context, self) - end - end - - return "" -end - -function renderer:_inverted(name, context, callback) - local value = context:lookup(name) - - -- From the spec: inverted sections may render text once based on the - -- inverse value of the key. That is, they will be rendered if the key - -- doesn't exist, is false, or is an empty list. - - if value == nil or value == false or (is_array(value) and #value == 0) then - return callback(context, self) - end - - return "" -end - -function renderer:_partial(name, context, originalTemplate) - local fn = self.partial_cache[name] - - -- check if partial cache exists - if (not fn and self.partials) then - - local partial = self.partials[name] - if (not partial) then - return "" - end - - -- compile partial and store result in cache - fn = self:compile(partial, nil, originalTemplate) - self.partial_cache[name] = fn - end - return fn and fn(context, self) or "" -end - -function renderer:_name(name, context, escape) - local value = context:lookup(name) - - if type(value) == "function" then - value = value(context.view) - end - - local str = value == nil and "" or value - str = tostring(str) - - if escape then - return string_gsub(str, '[&<>"\'/]', function(s) return html_escape_characters[s] end) - end - - return str -end - --- Breaks up the given `template` string into a tree of token objects. If --- `tags` is given here it must be an array with two string values: the --- opening and closing tags used in the template (e.g. ["<%", "%>"]). Of --- course, the default is to use mustaches (i.e. Mustache.tags). -function renderer:parse(template, tags) - tags = tags or self.tags - local tag_patterns = escape_tags(tags) - local scanner = Scanner:new(template) - local tokens = {} -- token buffer - local spaces = {} -- indices of whitespace tokens on the current line - local has_tag = false -- is there a {{tag} on the current line? - local non_space = false -- is there a non-space char on the current line? - - -- Strips all whitespace tokens array for the current line if there was - -- a {{#tag}} on it and otherwise only space - - local type, value, chr - - while not scanner:eos() do - local start = scanner.pos - - value = scanner:scan_until(tag_patterns[1]) - - if value then - for i = 1, #value do - chr = string_sub(value,i,i) - - if string_find(chr, "%s+") then - spaces[#spaces+1] = #tokens - else - non_space = true - end - - tokens[#tokens+1] = { type = "text", value = chr, startIndex = start, endIndex = start } - start = start + 1 - end - end - - if not scanner:scan(tag_patterns[1]) then - break - end - - has_tag = true - type = scanner:scan(patterns.tag) or "name" - - scanner:scan(patterns.white) - - if type == "=" then - value = scanner:scan_until(patterns.eq) - scanner:scan(patterns.eq) - scanner:scan_until(tag_patterns[2]) - elseif type == "{" then - local close_pattern = "%s*}"..tags[2] - value = scanner:scan_until(close_pattern) - scanner:scan(patterns.curly) - scanner:scan_until(tag_patterns[2]) - else - value = scanner:scan_until(tag_patterns[2]) - end - - if not scanner:scan(tag_patterns[2]) then - error("Unclosed tag at " .. scanner.pos) - end - - tokens[#tokens+1] = { type = type, value = value, startIndex = start, endIndex = scanner.pos - 1 } - if type == "name" or type == "{" or type == "&" then - non_space = true --> what does this do? - end - - if type == "=" then - tags = string_split(value, patterns.space) - tag_patterns = escape_tags(tags) - end - end - - return nest_tokens(squash_tokens(tokens)) -end - -function renderer:new() - local out = { - cache = {}, - partial_cache = {}, - tags = {"{{", "}}"} - } - return setmetatable(out, { __index = self }) -end - -return renderer diff --git a/src/bindings/lua/flux/lustache/scanner.lua b/src/bindings/lua/flux/lustache/scanner.lua deleted file mode 100644 index 0673df112305..000000000000 --- a/src/bindings/lua/flux/lustache/scanner.lua +++ /dev/null @@ -1,57 +0,0 @@ -local string_find, string_match, string_sub = - string.find, string.match, string.sub - -local scanner = {} - --- Returns `true` if the tail is empty (end of string). -function scanner:eos() - return self.tail == "" -end - --- Tries to match the given regular expression at the current position. --- Returns the matched text if it can match, `null` otherwise. -function scanner:scan(pattern) - local match = string_match(self.tail, pattern) - - if match and string_find(self.tail, pattern) == 1 then - self.tail = string_sub(self.tail, #match + 1) - self.pos = self.pos + #match - - return match - end - -end - --- Skips all text until the given regular expression can be matched. Returns --- the skipped string, which is the entire tail of this scanner if no match --- can be made. -function scanner:scan_until(pattern) - - local match - local pos = string_find(self.tail, pattern) - - if pos == nil then - match = self.tail - self.pos = self.pos + #self.tail - self.tail = "" - elseif pos == 1 then - match = nil - else - match = string_sub(self.tail, 1, pos - 1) - self.tail = string_sub(self.tail, pos) - self.pos = self.pos + #match - end - - return match -end - -function scanner:new(str) - local out = { - str = str, - tail = str, - pos = 1 - } - return setmetatable(out, { __index = self } ) -end - -return scanner diff --git a/src/bindings/lua/wreck.lua b/src/bindings/lua/wreck.lua deleted file mode 100644 index d2fa3223192b..000000000000 --- a/src/bindings/lua/wreck.lua +++ /dev/null @@ -1,844 +0,0 @@ -------------------------------------------------------------- --- Copyright 2014 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - -local posix = require 'flux.posix' -local hostlist = require 'flux.hostlist' -local cpuset = require 'flux.affinity'.cpuset - -local wreck = {} -wreck.__index = wreck; - -local lwj_options = { --- ['ntasks'] = "Set total number of tasks to execute", - ['commit-on-task-exit'] = "Call kvs_commit for each task exit", - ['stdio-delay-commit'] = "Don't call kvs_commit for each line of output", - ['stdio-commit-on-open'] = "Commit to kvs on stdio open in each task", - ['stdio-commit-on-close'] = "Commit to kvs on stdio close in each task", - ['nokz'] = "Do not store job output in kvs", - ['kz'] = "Force job output to kvs", - ['stop-children-in-exec'] = "Start tasks in STOPPED state for debugger", - ['no-pmi-server'] = "Do not start simple-pmi server", - ['trace-pmi-server'] = "Log simple-pmi server protocol exchange", - ['cpu-affinity'] = "Set default cpu-affinity to assigned cores", - ['gpubind'] = "Control CUDA_VISIBLE_DEVICES [=per-task,off]", - ['mpi'] = "Set hint for type of MPI, e.g. -o mpi=spectrum " -} - -local default_opts = { - ['help'] = { char = 'h' }, - ['name'] = { char = 'J', arg = "NAME" }, - ['verbose'] = { char = 'v' }, - ['ntasks'] = { char = 'n', arg = "N" }, - ['gpus-per-task'] = { char = 'g', arg = "g" }, - ['cores-per-task'] = { char = 'c', arg = "N" }, - ['nnodes'] = { char = 'N', arg = "N" }, - ['tasks-per-node'] = - { char = 't', arg = "N" }, - ['walltime'] = { char = "T", arg = "SECONDS" }, - ['output'] = { char = "O", arg = "FILENAME" }, - ['error'] = { char = "E", arg = "FILENAME" }, - ['input'] = { char = "i", arg = "HOW" }, - ['label-io'] = { char = "l", }, - ['skip-env'] = { char = "S", }, - ['epilog'] = { char = "x", arg = "SCRIPT" }, - ['postscript'] = - { char = "p", arg = "SCRIPT" }, - ['options'] = { char = 'o', arg = "OPTIONS.." }, -} - -local function opt_table (w) - local o = {} - for x,t in pairs (default_opts) do - o[x] = t.char - end - for _,v in pairs (w.extra_options) do - o[v.name] = v.char - end - return o -end - -local function short_opts (w) - local s = "" - for x,t in pairs (default_opts) do - s = s .. t.char .. (t.arg and ":" or "") - end - for _,v in pairs (w.extra_options) do - s = s .. v.char .. (v.arg and ":" or "") - end - return s -end - -function wreck:say (...) - io.stderr:write (self.prog..": "..string.format (...)) -end - -function wreck:verbose (...) - if self.opts.v then - self:say (...) - end -end - -function wreck:die (...) - self:say (...) - os.exit (1) -end - -function wreck:usage() - io.stderr:write ("Usage: "..self.prog.." OPTIONS.. COMMANDS..\n") - io.stderr:write ([[ - -h, --help Display this message - -v, --verbose Be verbose - -N, --name=NAME Set an optional name for job to NAME - -n, --ntasks=N Request to run a total of N tasks - -c, --cores-per-task=N Request N cores per task - -g, --gpus-per-task=N Request N GPUs per task - -N, --nnodes=N Force number of nodes - -t, --tasks-per-node=N Force number of tasks per node - -o, --options=OPTION,... Set other options (See OTHER OPTIONS below) - -T, --walltime=N[SUFFIX] Set max job walltime to N seconds. Optional - suffix may be 's' for seconds (default), 'm' - for minutes, 'h' for hours or 'd' for days. - N may be an arbitrary floating point number, - but will be rounded up to nearest second. - -O, --output=FILENAME Duplicate stdout/stderr from tasks to a file or - files. FILENAME is optionally a mustache - template with keys such as id, cmd, taskid. - (e.g. --output=flux-{{id}}.out) - -i, --input=HOW Indicate how to deal with stdin for tasks. HOW - is a list of [src:]dst pairs separated by semi- - colon, where src is an optional src file - (default: stdin) and dst is a list of taskids - or "*" or "all". E.g. (-i0 sends stdin to task 0, - -i /dev/null:* closes all stdin, etc.) - -E, --error=FILENAME Send stderr to a different location than stdout. - -l, --labelio Prefix lines of output with task id - -S, --skip-env Skip export of environment to job - -x, --epilog=PATH Execute a script after all tasks exit but before - the job state is set to "complete" - -p, --postscript=PATH Execute a script after job state is "complete" -]]) - for _,v in pairs (self.extra_options) do - local optstr = v.name .. (v.arg and "="..v.arg or "") - io.stderr:write ( - string.format (" -%s, --%-20s %s\n", v.char, optstr, v.usage)) - end - io.stderr:write ("\nOTHER OPTIONS:\n") - for o,d in pairs (lwj_options) do - io.stderr:write (string.format (" %-26s %s\n", o, d)) - end -end - - -function wreck.get_filtered_env () - local env = posix.getenv() - env.HOSTNAME = nil - env.ENVIRONMENT = nil - for k,v in pairs (env) do - if k:match ("SLURM_") then env[k] = nil end - if k:match ("FLUX_URI") then env[k] = nil end - end - return (env) -end - -local function get_job_env (arg) - local f = arg.flux - local env = wreck.get_filtered_env () - local default_env = {} - if f then default_env = f:kvs_get ("lwj.environ") or {} end - for k,v in pairs (env) do - -- If same key=value is already in default env no need to - -- export it again, remove: - if default_env[k] == env[k] then env[k] = nil end - end - return (env) -end - -local function array_tonumber (t) - for i = 1, #t do - t[i] = tonumber (t[i]) - end - return t -end - - -local function job_kvspath (f, id) - assert (id, "Required argument id missing!") - local arg = { id } - if type (id) == "table" then - arg = id - end - local r, err = f:rpc ("job.kvspath", {ids = array_tonumber (arg) }) - if not r then error (err) end - if type (id) == "table" then return r.paths end - return r.paths [1] -end - ---- --- Return kvs path to job id `id` --- -local kvs_paths = {} -local function kvs_path_insert (id, path) - kvs_paths [id] = path -end -local function kvs_path (f, id) - if not kvs_paths[id] then - kvs_path_insert (id, job_kvspath (f, id)) - end - return kvs_paths [id] -end - -local function kvs_path_multi (f, ids) - local result = job_kvspath (f, ids) - for i,id in ipairs (ids) do - kvs_path_insert (id, result [id]) - end - return result -end - -function wreck:lwj_path (id) - assert (id, "missing required argument `id'") - if not self.lwj_paths[id] then - self.lwj_paths [id] = job_kvspath (self.flux, id) - end - return self.lwj_paths [id] -end - -function wreck:kvsdir (id) - if not self.kvsdirs[id] then - local f = self.flux - local d, err = f:kvsdir (self:lwj_path (id)) - if not d then return nil, err end - self.kvsdirs[id] = d - end - return self.kvsdirs[id] -end - -function wreck:add_options (opts) - for k,v in pairs (opts) do - if not type (v) == "table" then return nil, "Invalid parameter" end - - local char = v.char - if default_opts [v.name] then - return nil, "Can't override option '"..k.."'" - end - table.insert (self.extra_options, v) - end - return true -end - -function wreck:getopt (opt) - if not self.opts then return nil end - return self.opts [opt] -end - -local function parse_walltime (s) - local m = { s = 1, m = 60, h = 3600, d = 56400 } - local n, suffix = s:match ("^([0-9.]+)([HhMmDdSs]?)$") - if not tonumber (n) then - return nil, "Invalid duration '"..s.."'" - end - if suffix and m [suffix] then - n = (n * m [suffix]) - end - return math.ceil (n) -end - -function wreck:initial_job_options () - if not self.flux then - local err - self.flux, err = require 'flux'.new () - if not self.flux then - self:die ("Unable to open connection to flux broker: %s", err) - end - end - return self.flux:kvs_get ("lwj.options") or {} -end - --- Create output configuration table from wreck object state, -function wreck:output_file_config () - if not self.opts.O and not self.opts.E then return nil end - return { - files = { - stdout = self.opts.O, - stderr = self.opts.E, - }, - labelio = self.opts.l, - } -end - -function wreck:parse_cmdline (arg) - local getopt = require 'flux.alt_getopt' .get_opts - local s = short_opts (self) - local v = opt_table (self) - self.opts, self.optind = getopt (arg, s, v) - - if self.opts.h then - self:usage() - os.exit (0) - end - - if self.opts.o then - for opt in self.opts.o:gmatch ("[^,]+") do - if not lwj_options [opt:match ("([^=]+)")] then - return nil, string.format ("Unknown LWJ option '%s'\n", opt) - end - end - end - - if self.opts.T then - self.walltime, err = parse_walltime (self.opts.T) - if not self.walltime then - self:die ("Error: %s", err) - end - end - - if self.optind > #arg then - self:say ("Error: remote command required\n") - self:usage() - os.exit (1) - end - - self.nnodes = self.opts.N and tonumber (self.opts.N) - - if not self.opts.t then - self.opts.t = 1 - end - - -- If nnodes was provided but -n, --ntasks not set, then - -- set ntasks to nnodes. - if self.opts.N and not self.opts.n then - self.ntasks = self.nnodes * self.opts.t - else - self.ntasks = self.opts.n and tonumber (self.opts.n) or 1 - end - if self.opts.c then - self.ncores = self.opts.c * self.ntasks - else - self.ncores = self.ntasks - end - - if self.opts.g then - self.ngpus = self.opts.g * self.ntasks - end - - self.tasks_per_node = self.opts.t - - self.cmdline = {} - for i = self.optind, #arg do - table.insert (self.cmdline, arg[i]) - end - - if not self.flux then - self.flux = require 'flux'.new() - end - - self.job_options = self:initial_job_options () - if self.opts.o then - for entry in self.opts.o:gmatch ('[^,]+') do - local opt, val = entry, 1 - if entry:match ("=") then - opt, val = entry:match ("(.+)=(.+)") - if tonumber(val) == 0 or val == "false" or val == "no" then - val = false - end - end - -- kz/nokz override global value of eachother - if opt == "kz" and self.job_options.nokz then - self.job_options.nokz = false - end - if opt == "nokz" and self.job_options.kz then - self.job_options.kz = false - end - self.job_options[opt] = val - end - end - - self.output = self:output_file_config () - return true -end - -local function inputs_table_from_args (wreck, s) - -- Default is to broadcast stdin to all tasks: - if not s then return { { src = "stdin", dst = "*" } } end - - local inputs = {} - for m in s:gmatch ("[^;]+") do - local src,dst = m:match ("^([^:]-):?([^:]+)$") - if not src or not dst then - wreck:die ("Invalid argument to --input: %s\n", s) - end - -- A dst of "none" short circuits rest of entries - if dst == "none" then return inputs end - - -- Verify the validity of dst - if dst ~= "all" and dst ~= "*" then - local h, err = cpuset.new (dst) - if not h then - if src ~= "" then - wreck:die ("--input: invalid dst: %s\n", dst) - end - -- otherwise, assume only dst was provided - src = dst - dst = "*" - end - end - if src == "" or src == "-" then src = "stdin" end - table.insert (inputs, { src = src, dst = dst }) - end - -- Add fall through option - table.insert (inputs, { src = "/dev/null", dst = "*" }) - return inputs -end - -local function fixup_nnodes (wreck) - if not wreck.flux then return end - if not wreck.nnodes then - wreck.nnodes = math.min (wreck.ntasks, wreck.flux.size) - elseif wreck.nnodes > wreck.flux.size then - wreck:die ("Requested nodes (%d) exceeds available (%d)\n", - wreck.nnodes, wreck.flux.size) - end -end - -local function random_string () - local f = assert (io.open ("/dev/urandom", "rb")) - local d = f:read(8) - f:close() - local s = "" - for i = 1, d:len() do - s = s..string.format ("%02x", d:byte(i)) - end - return (s) -end - -function wreck:setup_ioservices () - -- Setup stderr/stdout ioservice ranks. Either flux-wreckrun - -- or nodeid 0 of the job will offer one or both of these services: - self.ioservices = {} - if self.job_options.nokz then - -- Default: register stdout and stderr service on this rank - local name = random_string () - self.ioservices = { - stdout = { - name = name..":out", - rank = self.flux.rank - }, - stderr = { - name = name..":err", - rank = self.flux.rank - } - } - if self.output then - -- Shunt stdio to nodeid 0 of the job if any output is being - -- redirected to files via the output.lua plugin: - local outfile = self.output.files.stdout - local errfile = self.output.files.stderr - if outfile then - self.ioservices.stdout.rank = -1 - end - if errfile or outfile then - self.ioservices.stderr.rank = -1 - end - end - end - return self.ioservices -end - - -function wreck:jobreq () - if not self.opts then return nil, "Error: cmdline not parsed" end - if self.fixup_nnodes then - fixup_nnodes (self) - end - local jobreq = { - name = self.opts.J, - nnodes = self.nnodes or 0, - ntasks = self.ntasks, - ncores = self.ncores, - cmdline = self.cmdline, - environ = self.opts.S and {} or get_job_env { flux = self.flux }, - cwd = posix.getcwd (), - walltime =self.walltime or 0, - ngpus = self.ngpus or 0, - - ["opts.nnodes"] = self.opts.N, - ["opts.ntasks"] = self.opts.n, - ["opts.cores-per-task"] = self.opts.c, - ["opts.gpus-per-task"] = self.opts.g, - ["opts.tasks-per-node"] = self.opts.t, - ["epilog.pre"] = self.opts.x, - ["epilog.post"] = self.opts.p, - } - jobreq.options = self.job_options - jobreq.output = self.output - jobreq ["input.config"] = inputs_table_from_args (self, self.opts.i) - - jobreq.ioservice = self:setup_ioservices () - - return jobreq -end - -local function send_job_request (self, name) - if not self.flux then - local f, err = require 'flux'.new () - if not f then return nil, err end - self.flux = f - end - local jobreq, err = self:jobreq () - if not jobreq then - return nil, err - end - local resp, err = self.flux:rpc (name, jobreq) - if not resp then - return nil, "flux.rpc: "..err - end - if resp.errnum then - return nil, name.." message failed with errnum="..resp.errnum - end - return resp -end - -function wreck:submit () - local resp, err = send_job_request (self, "job.submit") - if not resp then return nil, err end - if resp.state ~= "submitted" then - return nil, "job submission failure, job left in state="..resp.state - end - return resp.jobid, resp.kvs_path -end - -function wreck:createjob () - self.fixup_nnodes = true - local resp, err = send_job_request (self, "job.create") - if not resp then return nil, err end - -- - -- If reply contains a kvs path to this LWJ, pre-memoize the id - -- to kvs path mapping so we do not have to fetch it again. - if resp.kvs_path then - self.lwj_paths [resp.jobid] = resp.kvs_path - kvs_paths [resp.jobid] = resp.kvs_path - end - return resp.jobid, resp.kvs_path -end - -local function initialize_args (arg) - if arg.ntasks and arg.nnodes then return true end - local f = arg.flux - local lwj, err = f:kvsdir (kvs_path (f, arg.jobid)) - if not lwj then - return nil, "Error: "..err - end - arg.ntasks = lwj.ntasks - arg.nnodes = lwj.nnodes - return true -end - -function wreck.ioattach (arg) - local ioplex = require 'wreck.io' - local f = arg.flux - local rc, err = initialize_args (arg) - if not rc then return nil, err end - - local taskio, err = ioplex.create (arg) - if not taskio then return nil, err end - for i = 0, arg.ntasks - 1 do - for _,stream in pairs {"stdout", "stderr"} do - local rc, err = taskio:redirect (i, stream, stream) - end - end - taskio:start (arg.flux) - return taskio -end - -local logstream = {} -logstream.__index = logstream - -local function logstream_print_line (iow, line) - io.stderr:write (string.format ("rank%d: Error: %s\n", iow.id, line)) -end - -function logstream:dump () - for _, iow in pairs (self.watchers) do - local r, err = iow.kz:read () - while r and r.data and not r.eof do - logstream_print_line (iow, r.data) - r, err = iow.kz:read () - end - end -end - -function wreck.logstream (arg) - local l = {} - local f = arg.flux - if not f then return nil, "flux argument member required" end - local rc, err = initialize_args (arg) - if not rc then return nil, err end - if not arg.nnodes then return nil, "nnodes argument required" end - l.watchers = {} - for i = 0, arg.nnodes - 1 do - local key = kvs_path (f, arg.jobid)..".log."..i - local iow, err = f:iowatcher { - key = key, - kz_flags = arg.oneshot and { "nofollow" }, - handler = function (iow, r, err) - if not r then - if err then - io.stderr:write ("logstream kz error "..err.."\n") - end - return - end - logstream_print_line (iow, r) - end - } - iow.id = i - table.insert (l.watchers, iow) - end - return setmetatable (l, logstream) -end - -local function exit_message (t) - local s = "exited with " - s = s .. (t.exit_code and "exit code" or "signal") .. " %d" - return s:format (t.exit_code or t.exit_sig) -end - -local function task_status (lwj, taskid) - if not tonumber (taskid) then return nil end - local t = lwj[taskid] - if not t.exit_status then - return 0, "running" - end - local x = t.exit_code or (t.exit_sig + 128) - return x, exit_message (t) -end - -local function exit_status_aggregate (arg) - local flux = require 'flux' - local f = arg.flux - local lwj = arg.kvsdir - local msgs = {} - - local exit_status = lwj.exit_status - if not exit_status then return nil, nil end - - local s, max, core = flux.exitstatus (exit_status.max) - if s == "killed" then max = max + 128 end - - for ids,v in pairs (exit_status.entries) do - local msg, code, core = flux.exitstatus (v) - s = "exited with " - s = s .. (msg == "exited" and "exit code " or "signal ") .. code - msgs [s] = hostlist.new (ids) - end - - return max, msgs -end - - ---- Return highest exit code from all tasks and task exit message list. --- Summarize job exit status for arg.jobid by returning max task exit code, --- along with a list of task exit messages to be optionally emitted to stdout. --- @param arg.jobid job identifier --- @param arg.flux (optional) flux handle --- @return exit_cde, msg_list -function wreck.status (arg) - local flux = require 'flux' - local f = arg.flux - local jobid = arg.jobid - if not jobid then return nil, "required arg jobid" end - - if not f then f = require 'flux'.new() end - local lwj = f:kvsdir (kvs_path (f, jobid)) - - local max = 0 - local msgs = {} - - if lwj.exit_status then - return exit_status_aggregate { flux = f, kvsdir = lwj } - end - - for taskid in lwj:keys () do - local x, s = task_status (lwj, taskid) - if x then - if x > max then max = x end - if not msgs[s] then - msgs[s] = hostlist.new (taskid) - else - msgs[s]:concat (taskid) - end - end - end - return max, msgs -end - -function wreck.id_to_path (arg) - local flux = require 'flux' - local f = arg.flux - local id = arg.jobid - if not id then return nil, "required arg jobid" end - if not f then f, err = require 'flux'.new () end - if not f then return nil, err end - return kvs_path (f, id) -end - -local function kvsdir_reverse_keys (dir) - local result = {} - for k in dir:keys () do - if tonumber (k) then - table.insert (result, k) - end - end - table.sort (result, function (a,b) return tonumber(b) < tonumber(a) end) - return result -end - -local function reverse (t) - local len = #t - local r = {} - for i = len, 1, -1 do - table.insert (r, t[i]) - end - return r -end - --- append array t2 to t1 -local function append (t1, t2) - for _,v in ipairs (t2) do table.insert (t1, v) end - return t1 -end - -function wreck.jobids_to_kvspath (arg) - local f = arg.flux - local ids = arg.jobids - if not ids then return nil, 'missing required arg jobids' end - if not f then f, err = require 'flux'.new () end - if not f then return nil, err end - return kvs_path_multi (f, ids) -end - -local function include_state (conf, state) - -- Include all states if no state conf is supplied - if not conf then return true end - - -- Otherwise, if there is a conf.include hash then *only* include - -- states that appear in this hash: - if conf.include then return conf.include[state] end - - -- Otherwise, exclude this state if it is in the conf.exclude hash: - if conf.exclude then return not conf.exclude[state] end - - -- No conf.include, no conf.exclude, I guess it is included: - return true -end - --- Convert keys of table t to a comma-separated list of strings -local function to_string_list (t) - if not t then return nil end - local r = {} - for k,v in pairs (t) do - if v then table.insert (r, k) end - end - return table.concat (r, ",") -end - --- Return a list and hash of active jobs by kvs path -local function joblist_active (arg) - local f = arg.flux - local conf = arg.states - local r, err = f:rpc ("job.list", - { max = arg.max, - include = to_string_list (conf and conf.include), - exclude = to_string_list (conf and conf.exclude) - }) - if not r then return nil, nil, err end - local active = {} - local results = {} - for _,job in ipairs (r.jobs) do - local path = job.kvs_path - kvs_path_insert (job.jobid, path) - active[path] = true - table.insert (results, path) - end - return results, active -end - -local function joblist_kvs (arg, exclude) - local flux = require 'flux' - local f = arg.flux - if not f then f, err = flux.new () end - if not f then return nil, err end - - local function visit (d, r) - local results = r or {} - if not d then return end - local dirs = kvsdir_reverse_keys (d) - for _,k in pairs (dirs) do - local path = tostring (d) .. "." .. k - local dir = f:kvsdir (path) - if dir and (not exclude or not exclude[path]) then - if dir.state then - -- This is a lwj dir, add to results table: - if include_state (arg.states, dir.state) then - table.insert (results, path) - end - else - -- recurse to find lwj dirs lower in the directory tree - visit (dir, results) - end - if arg.max and #results >= arg.max then return results end - end - end - return results - end - - local dir, err = f:kvsdir ("lwj") - if not dir then - if err:match ("No such") then err = "No job information in KVS" end - return nil, err - end - - return reverse (visit (dir)) -end - -function wreck.joblist (arg) - local results, active = {}, {} - if not arg.kvs_only then - results, active = joblist_active (arg) - if results and arg.max then - arg.max = arg.max - #results - end - if arg.active_only or arg.max == 0 then return results end - end - - -- Append a list of jobs from the kvs to the active jobs - local r, err = joblist_kvs (arg, active) - if not r then return nil, err end - - return append (results or {}, r) -end - -local function shortprog () - local prog = string.match (arg[0], "([^/]+)$") - return prog:match ("flux%-(.+)$") -end - -function wreck.new (arg) - if not arg then arg = {} end - local w = setmetatable ({ extra_options = {}, - kvsdirs = {}, - lwj_paths = {}}, wreck) - w.prog = arg.prog or shortprog () - w.flux = arg.flux - return w -end - -return wreck - --- vi: ts=4 sw=4 expandtab diff --git a/src/bindings/lua/wreck/io.lua b/src/bindings/lua/wreck/io.lua deleted file mode 100644 index f22298a85309..000000000000 --- a/src/bindings/lua/wreck/io.lua +++ /dev/null @@ -1,369 +0,0 @@ -------------------------------------------------------------- --- Copyright 2014 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - --- --- Task I/O library for WRECK --- - - --- --- ostream - simple output stream with reference count. --- -local ostream = {} -ostream.__index = ostream -function ostream.create (filename) - local s = { count = 0, filename = filename, flag = "w" } - setmetatable (s, ostream) - return s -end - -function ostream:closed () - return self.count == 0 -end - -function ostream:open () - if not self.fp then - if self.filename == "stdout" or self.filename == "stderr" then - self.fp = io[self.filename] - else - self.fp, err = io.open (self.filename, self.flag) - self.fp:setvbuf 'line' - end - if not self.fp then return nil, self.filename..": "..err end - end - self.count = self.count + 1 - return true -end - -function ostream:close () - self.count = self.count - 1 - if self.count == 0 then - self.fp:close () - self.fp = nil - end -end - -function ostream:write (data) - if not self.fp then return nil, "File not open" end - local rc, err = self.fp:write (data) - if not rc then print (err) end -end - - -local kvsfile = {} -kvsfile.__index = kvsfile - -function kvsfile.create (arg) - local s = { count = 0, key = arg.key, flux = arg.flux, data = "" } - setmetatable (s, kvsfile) - return s -end - -function kvsfile:open () - self.count = self.count + 1 - return true -end - -function kvsfile:close () - self.count = self.count - 1 - if self.count == 0 then - local rc, err = self.flux:kvs_put (self.key, self.data) - if not rc then io.stderr:write (err) end - self.flux:kvs_commit () - end -end - -function kvsfile:closed () - return self.count == 0 -end - -function kvsfile:write (data) - self.data = self.data .. data -end - --- --- wreck job io tracking object. --- Direct all task io to a named set of ostream objects --- -local ioplex = {} -ioplex.__index = ioplex - - ---- Create a wreck i/o multiplexer --- ioplex objects are an abstraction of wreck reactor-driven I/O --- with many-to-one or many-to-many output redirection. Named --- output stream objects are registered with an ioplex object, --- and then task "stdout" and "stderr" streams are redirected --- onto these output streams. Output streams have reference counting --- and are only closed when all references have been closed. --- When all outputs are closed ioplex:complete() will return true. ---@param arg a Table of named keys for arguments ---@return ioplex object --- -function ioplex.create (arg) - if not arg.flux or not arg.jobid then - return nil, "Required handle or jobid arg missing!" - end - - local io = { - flux = arg.flux, - id = arg.jobid, - labelio = arg.labelio, - kvspath = arg.kvspath, - on_completion = arg.on_completion, - log_err = arg.log_err, - nofollow = arg.nofollow, - nokz = arg.nokz, - removed = {}, - output = {}, - files = {} - } - if not io.kvspath then - local wreck = require 'wreck' - local r, err = wreck.id_to_path { flux = io.flux, jobid = io.id } - if not r then error (err) end - io.kvspath = r - end - if io.nokz then - if not arg.ioservices then return nil, "required arg ioservices" end - io.ioservices = {} - for s,v in pairs (arg.ioservices) do - if v and v.rank == io.flux.rank then - io.ioservices[s] = v.name - end - end - end - setmetatable (io, ioplex) - return io -end - -local function eprintf (...) - io.stderr:write (string.format (...).."\n") -end - -function ioplex:enable_debug (logger) - self.logger = logger and logger or eprintf -end - -function ioplex:log (fmt, ...) - if self.logger then - self.logger (fmt, ...) - end -end - -function ioplex:err (fmt, ...) - if self.log_err then - self.log_err (fmt, ...) - else - io.stderr:write (string.format (fmt, ...)) - end -end - - -local function ioplex_set_stream (self, taskid, name, f) - if not self.output[taskid] then - self.output[taskid] ={} - end - self.output[taskid][name] = f -end - -local function ioplex_open_path (self, path) - local f, err - local key = path:match ("^kvs://(.+)$") - if not key then - self:log ("creating path %s", path) - return ostream.create (path) - else - self:log ("opening kvs key %s", key) - return kvsfile.create { flux = self.flux, key = key } - end -end - -local function ioplex_create_stream (self, path) - local files = self.files - if not files[path] then - local f, err = ioplex_open_path (self, path) - if not f then return nil, err end - files[path] = f - end - return files[path] -end - -function ioplex:close_output (of) - of:close () - if of:closed () then - self:log ("closed path %s", of.filename) - end - if self:complete() and self.on_completion then - self.on_completion() - end -end - -function ioplex:output_handler (arg) - local taskid = arg.taskid or 0 - local stream = arg.stream or "stdout" - local data = arg.data - local of = arg.output or self.output[taskid][stream] - - if not data then - self:close_output (of) - return - end - if self.labelio then - of:write (taskid..": ") - end - of:write (data) -end - -function ioplex:register_services () - local f = self.flux - if not self.ioservices then return nil, "IO Service not started yet!" end - - for _,service in pairs (self.ioservices) do - local rc, err = f:rpc ("service.add", { service = service }) - if not rc then - return nil, err - end - end - return true -end - -function ioplex:msg_handler (stream, zmsg) - local data = nil - local msg = zmsg.data - if not msg then return end - - if not msg.eof then - data = msg.data - end - self:output_handler { taskid = msg.taskid, - data = data, - stream = stream } -end - -function ioplex:ioservice_start () - -- I/O services for stdout/err for this job: - for stream,service in pairs (self.ioservices) do - local mh, err = self.flux:msghandler { - pattern = service..".write", - handler = function (f, zmsg, mh) - self:msg_handler (stream, zmsg) - end - } - if not mh then self:err ("ioservice_start: %s", err) end - end -end - -local function ioplex_taskid_start (self, flux, taskid, stream) - local of = self.output[taskid][stream] - if not of then return nil, "No stream "..stream.." for task " .. taskid end - - local flags = {} - if self.nofollow then table.insert (flags, "nofollow") end - - local f = flux - local key = string.format ("%s.%d.%s", self.kvspath, taskid, stream) - local iow, err = f:iowatcher { - key = key, - kz_flags = flags, - handler = function (iow, data, err) - self:output_handler { taskid = taskid, - stream = stream, - output = of, - data = data } - end - } - if not iow then - self:log ("ignoring %s: %s", key, err) - -- remove reference count - of:close () - end -end - -function ioplex:closeall (stream) - for taskid, t in pairs (self.output) do - if t[stream] then t[stream]:close () end - end -end - -function ioplex:close_unwatched_streams () - for _,stream in pairs { "stdout", "stderr" } do - if not self.ioservices [stream] then self:closeall (stream) end - end -end - ---- "start" all ioplex iowatchers on reactor. --- @param h flux handle of reactor to use (optional) -function ioplex:start (h) - local flux = h or self.flux - if not flux then return nil, "Error: no flux handle!" end - - if self.nokz then - self:ioservice_start () - local rc, err = self:register_services () - if not rc then - self:err ("Unable to start I/O service: %s", err) - end - self:close_unwatched_streams () - return - end - - for taskid, t in pairs (self.output) do - for stream, f in pairs (t) do - ioplex_taskid_start (self, flux, taskid, stream) - end - end - self.started = true -end - ---- redirect a stream from a task to named stream "path". --- On first reference to `path` (not including special paths "stdout" --- and "stderr") the referenced file will be opened with `io.open(..., "w")` --- --- @param taskid task identifier --- @param stream name of stream to redirect (e.g. "stdout") --- @param path path of file to redirect to (or "stderr" "stdout") --- @return true on success, nil with error on failure --- -function ioplex:redirect (taskid, stream, path) - local f, err = ioplex_create_stream (self, path) - if not f then return nil, err end - - ioplex_set_stream (self, taskid, stream, f) - return f:open() -end - ---- duplicate a stream from a task onto existing stream for this task. --- Reference count is incremented for `dst` --- @param taskid task identifier --- @param src the source stream to duplicate (e.g. "stderr") --- @param dst the destination stream to copyto (e.g. "stdout") --- -function ioplex:dup (taskid, src, dst) - local f = self.output[taskid][dst] - if not f then - return nil, 'No such output stream: '..dst - end - self.output[taskid][src] = f - return f:open() -end - ---- Check for completion of I/O on an ioplex object --- @return true if IO is complete, false if not -function ioplex:complete () - for _,f in pairs (self.files) do - if not f:closed() then return false end - end - return true -end - -return ioplex - --- vi: ts=4 sw=4 expandtab diff --git a/src/bindings/python/flux/Makefile.am b/src/bindings/python/flux/Makefile.am index 03148eb206bc..10c5c071ec20 100644 --- a/src/bindings/python/flux/Makefile.am +++ b/src/bindings/python/flux/Makefile.am @@ -5,7 +5,6 @@ fluxpy_PYTHON=\ rpc.py\ message.py\ constants.py\ - jsc.py\ kz.py\ job.py \ mrpc.py \ diff --git a/src/bindings/python/flux/__init__.py b/src/bindings/python/flux/__init__.py index 0dde744db3d0..4e652dd25313 100644 --- a/src/bindings/python/flux/__init__.py +++ b/src/bindings/python/flux/__init__.py @@ -19,4 +19,4 @@ def Flux(*args, **kwargs): return flux.core.handle.Flux(*args, **kwargs) -__all__ = ["core", "kvs", "jsc", "rpc", "mrpc", "constants", "Flux"] +__all__ = ["core", "kvs", "rpc", "mrpc", "constants", "Flux"] diff --git a/src/bindings/python/flux/jsc.py b/src/bindings/python/flux/jsc.py deleted file mode 100644 index f3e5aec3871d..000000000000 --- a/src/bindings/python/flux/jsc.py +++ /dev/null @@ -1,89 +0,0 @@ -############################################################### -# Copyright 2014 Lawrence Livermore National Security, LLC -# (c.f. AUTHORS, NOTICE.LLNS, COPYING) -# -# This file is part of the Flux resource manager framework. -# For details, see https://github.com/flux-framework. -# -# SPDX-License-Identifier: LGPL-3.0 -############################################################### - -import json -import re -import sys - -import six - -from flux.wrapper import Wrapper -from _flux._core import ffi, lib - -MOD = sys.modules[__name__] -# Inject enum/define names matching ^JSC_[A-Z_]+$ into module -PATTERN = re.compile("^JSC_[A-Z_]+") -for k in dir(lib): - if PATTERN.match(k): - v = ffi.string(getattr(lib, k)).decode("ascii") - setattr(MOD, k, v) - - -class JSCWrapper(Wrapper): - """ - Generic JSC wrapper - """ - - def __init__(self): - """Set up the wrapper interface for functions prefixed with jsc_""" - super(JSCWrapper, self).__init__(ffi, lib, prefixes=["jsc_"]) - - -RAW = JSCWrapper() - - -def query_jcb(flux_handle, jobid, key): - jcb_str = ffi.new("char *[1]") - RAW.query_jcb(flux_handle, jobid, key, jcb_str) - if jcb_str[0] == ffi.NULL: - return None - return json.loads(ffi.string(jcb_str[0]).decode("utf-8")) - - -def update_jcb(flux_handle, jobid, key, jcb): - # TODO: validate the key is one of the constants in jstatctl.h - if isinstance(jcb, six.text_type): - jcb = jcb.encode("utf-8") - elif not isinstance(jcb, six.binary_type): - jcb = json.dumps(jcb).encode("utf-8") - return RAW.jsc_update_jcb(flux_handle, jobid, key, jcb) - - -@ffi.callback("jsc_handler_f") -def jsc_notify_wrapper(jcb, arg, errnum): - if jcb != ffi.NULL: - jcb = ffi.string(jcb).decode("utf-8") - callback, real_arg = ffi.from_handle(arg) - ret = callback(jcb, real_arg, errnum) - return ret if ret is not None else 0 - - -HANDLES = [] - - -def notify_status(flux_handle, fun, arg): - warg = (fun, arg) - whandle = ffi.new_handle(warg) - HANDLES.append(whandle) - return RAW.notify_status(flux_handle, jsc_notify_wrapper, whandle) - - -def job_num2state(job_state): - ret = RAW.job_num2state(job_state) - if ret == ffi.NULL: - return None - return ret.decode("ascii") - - -def job_state2num(job_state): - if isinstance(job_state, six.text_type): - # jsc doesn't use utf-8 internally, catch erroneous unicode here - job_state.encode("ascii") - return RAW.job_state2num(job_state) diff --git a/src/broker/broker.c b/src/broker/broker.c index dff1a0d52957..528e477f0345 100644 --- a/src/broker/broker.c +++ b/src/broker/broker.c @@ -686,8 +686,6 @@ static struct attrmap attrmap[] = { { "FLUX_PMI_LIBRARY_PATH", "conf.pmi_library_path", 1 }, { "FLUX_RC1_PATH", "broker.rc1_path", 1 }, { "FLUX_RC3_PATH", "broker.rc3_path", 1 }, - { "FLUX_WRECK_LUA_PATTERN", "wrexec.lua_pattern", 1 }, - { "FLUX_WREXECD_PATH", "wrexec.wrexecd_path", 1 }, { "FLUX_SEC_DIRECTORY", "security.keydir", 1 }, { "FLUX_URI", "parent-uri", 0 }, @@ -699,8 +697,6 @@ static void init_attrs_from_environment (attr_t *attrs) struct attrmap *m; const char *val; int flags = 0; // XXX possibly these should be immutable? - // however they weren't before and wreck test depends - // on changing wrexec.lua_pattern for (m = &attrmap[0]; m->env != NULL; m++) { val = getenv (m->env); diff --git a/src/cmd/.gitignore b/src/cmd/.gitignore index f58fa42ec295..bf78335bd263 100644 --- a/src/cmd/.gitignore +++ b/src/cmd/.gitignore @@ -3,7 +3,6 @@ /flux-comms-stats /flux-event /flux-jobspec-validate -/flux-jstat /flux-keygen /flux-kvs /flux-logger diff --git a/src/cmd/Makefile.am b/src/cmd/Makefile.am index ad6d9fef5ab5..35907aaef87d 100644 --- a/src/cmd/Makefile.am +++ b/src/cmd/Makefile.am @@ -54,9 +54,6 @@ nodist_flux_SOURCES = \ # dist_fluxcmd_SCRIPTS = \ - flux-submit \ - flux-wreckrun \ - flux-wreck \ flux-cron \ flux-aggregate \ flux-hostlist @@ -70,7 +67,6 @@ fluxcmd_PROGRAMS = \ flux-comms \ flux-kvs \ flux-start \ - flux-jstat \ flux-job \ flux-exec diff --git a/src/cmd/flux-hostlist b/src/cmd/flux-hostlist index 3ba18fc5b0fe..f568c577b486 100755 --- a/src/cmd/flux-hostlist +++ b/src/cmd/flux-hostlist @@ -11,12 +11,11 @@ ------------------------------------------------------------- -- --- flux-hostlist: print list of hosts in hostfile format for instance or job +-- flux-hostlist: print list of hosts in hostfile format for instance -- -- local getopt = require 'flux.alt_getopt'.get_opts local hostlist = require 'flux.hostlist' -local wreck = require 'wreck' local prog = "flux-hostlist" local hostkey = "resource.hosts" @@ -71,61 +70,6 @@ local function instance_host_table (arg) return hosts end --- Get R_lite at arg.kvspath and return comma-separated list of hostnames -local function R_lite_to_hosts (arg) - local f = arg.flux - local dir = arg.kvspath - local hosts = {} - local allhosts - - local R, err = f:kvs_get (dir..".R_lite") - if not R then die ("Failed to get R_lite at %s: %s\n", dir, err) end - - for _,entry in ipairs (R) do - if arg.ranks then - table.insert (hosts, entry.rank) - elseif entry.node then - table.insert (hosts, entry.node) - else - local allhosts = instance_host_table { flux = f } - table.insert (hosts, allhosts [entry.rank+1]) - end - end - return table.concat (hosts, ",") -end - --- Return a list of hosts for jobids given in arg.args list. -local function jobs_host_table (arg) - local f = arg.flux - local args = arg.args - local ranks = arg.ranks - local function toint (t) - local r = {} - for _,v in ipairs (t) do table.insert (r, tonumber (v)) end - return r - end - local function bracketify (t) - local r = {} - for _,v in ipairs (t) do table.insert (r, "["..v.."]") end - return r - end - local hl, err = hostlist.union (unpack (bracketify (args))) - if not hl then die ("hostlist.union: %s\n", err) end - - local kvspaths = wreck.jobids_to_kvspath { flux = f, - jobids = toint (hl:expand()) - } - if not kvspaths then die ("wreck.jobids_to_kvspath failed") end - - local hosts = hostlist.new () - - for _,p in ipairs (kvspaths) do - hosts:concat (R_lite_to_hosts { flux = f, kvspath = p, ranks = ranks }) - end - return hosts:expand() -end - - local function hosts_from_stream (fp) local hosts = {} for line in fp:lines() do @@ -164,8 +108,8 @@ local function set_hostlist (arg) end local function usage (code) - printf ("Usage: %s [OPTIONS] [JOBIDS]...\n", prog) - printf ("Get/set a list of hosts for the current session or job(s)\n") + printf ("Usage: %s [OPTIONS]...\n", prog) + printf ("Get/set a list of hosts for the current session\n") printf ("Options:\n") printf (" -h, --help Display this message.\n") printf (" -r, --ranks Output rank ids instead of hostnames.\n") @@ -199,14 +143,8 @@ end for i = 1,optind-1 do table.remove (arg, 1) end --- Otherwise, read either the instance hosts table or job --- specific hosts lists and emit one host per line to stdout: -local hosts = {} -if #arg == 0 then - hosts = instance_host_table { flux = f, ranks = opts.r } -else - hosts = jobs_host_table { flux = f, ranks = opts.r, args = arg } -end +-- read the instance hosts table +local hosts = instance_host_table { flux = f, ranks = opts.r } -- Print result: if opts.c then diff --git a/src/cmd/flux-jstat.c b/src/cmd/flux-jstat.c deleted file mode 100644 index 35dc555458a4..000000000000 --- a/src/cmd/flux-jstat.c +++ /dev/null @@ -1,270 +0,0 @@ -/************************************************************\ - * Copyright 2014 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "src/common/libutil/log.h" -#include "src/common/libutil/xzmalloc.h" - - -/****************************************************************************** - * * - * Internal types, macros and static variables * - * * - ******************************************************************************/ -typedef struct { - flux_t *h; - FILE *op; -} jstatctx_t; - -/* Note: Should only be used for signal handling */ -static flux_t *sig_flux_h; - -#define OPTIONS "o:h" -static const struct option longopts[] = { - {"help", no_argument, 0, 'h'}, - {"testout", required_argument, 0, 'o'}, - { 0, 0, 0, 0 }, -}; - - -/****************************************************************************** - * * - * Utilities * - * * - ******************************************************************************/ -static void usage (int code) -{ - fprintf (stderr, -"Usage: flux-jstat notify\n" -" flux-jstat query jobid \n" -" flux-jstat update jobid \n" -); - exit (code); -} - -static void freectx (void *arg) -{ - jstatctx_t *ctx = arg; - if (ctx->op) - fclose (ctx->op); -} - -static jstatctx_t *getctx (flux_t *h) -{ - jstatctx_t *ctx = (jstatctx_t *)flux_aux_get (h, "jstat"); - if (!ctx) { - ctx = xzmalloc (sizeof (*ctx)); - ctx->h = h; - ctx->op = NULL; - flux_aux_set (h, "jstat", ctx, freectx); - } - return ctx; -} - -static void sig_handler (int s) -{ - if (s == SIGINT) { - fprintf (stdout, "Exit on INT"); - exit (0); - } -} - -static FILE *open_test_outfile (const char *fn) -{ - FILE *fp; - if (!fn) - fp = NULL; - else if ( !(fp = fopen (fn, "w"))) - fprintf (stderr, "Failed to open %s\n", fn); - return fp; -} - -/****************************************************************************** - * * - * Async notification callback * - * * - ******************************************************************************/ - -static int job_status_cb (const char *jcbstr, void *arg, int errnum) -{ - int os; - int ns; - jstatctx_t *ctx = NULL; - flux_t *h = (flux_t *)arg; - json_t *jcb = NULL; - - ctx = getctx (h); - if (errnum > 0) { - flux_log (ctx->h, LOG_ERR, "job_status_cb: errnum passed in"); - return -1; - } - - if (!(jcb = json_loads (jcbstr, 0, NULL)) - || json_unpack (jcb, "{s:{s:i s:i}}", JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, &os, - JSC_STATE_PAIR_NSTATE, &ns) < 0) { - flux_log (ctx->h, LOG_ERR, "job_status_cb: error parsing JSON string"); - json_decref (jcb); - return -1; - } - - fprintf (ctx->op, "%s->%s\n", jsc_job_num2state (os), - jsc_job_num2state (ns)); - fflush (ctx->op); - json_decref (jcb); - return 0; -} - - -/****************************************************************************** - * * - * Top-level Handlers for each of notify, query and update commands * - * * - ******************************************************************************/ - -static int handle_notify_req (flux_t *h, const char *ofn) -{ - jstatctx_t *ctx = NULL; - - sig_flux_h = h; - if (signal (SIGINT, sig_handler) == SIG_ERR) - return -1; - - ctx = getctx (h); - ctx->op = (ofn)? open_test_outfile (ofn) : stdout; - - if (jsc_notify_status (h, job_status_cb, (void *)h) != 0) { - flux_log (h, LOG_ERR, "failed to reg a job status change CB"); - return -1; - } - if (flux_reactor_run (flux_get_reactor (h), 0) < 0) - flux_log (h, LOG_ERR, "error in flux_reactor_run"); - - return 0; -} - -static int handle_query_req (flux_t *h, int64_t j, const char *k, const char *n) -{ - jstatctx_t *ctx = getctx (h); - json_t *jcb = NULL; - char *jcbstr = NULL; - char *jcbstr_pretty = NULL; - - ctx->op = n ? open_test_outfile (n) : stdout; - if (jsc_query_jcb (h, j, k, &jcbstr) != 0) { - flux_log (h, LOG_ERR, "jsc_query_jcb reported an error"); - return -1; - } - if (!(jcb = json_loads (jcbstr, 0, NULL))) { - flux_log (h, LOG_ERR, "error loading jcbstr"); - goto done; - } - if (!(jcbstr_pretty = json_dumps (jcb, JSON_INDENT(2)))) { - flux_log (h, LOG_ERR, "error dumping jcbstr"); - goto done; - } - - fprintf (ctx->op, "Job Control Block: attribute %s for job %"PRIi64"\n", k, j); - fprintf (ctx->op, "%s\n", jcbstr_pretty); -done: - free (jcbstr_pretty); - json_decref (jcb); - free (jcbstr); - return 0; -} - -static int handle_update_req (flux_t *h, int64_t j, const char *k, - const char *jcbstr, const char *n) -{ - jstatctx_t *ctx = NULL; - ctx = getctx (h); - ctx->op = n? open_test_outfile (n) : stdout; - - return (jsc_update_jcb (ctx->h, j, k, jcbstr)); -} - - -/****************************************************************************** - * * - * Main entry point * - * * - ******************************************************************************/ - -int main (int argc, char *argv[]) -{ - flux_t *h; - int ch = 0; - int rc = 0; - char *cmd = NULL; - const char *j = NULL; - const char *ofn = NULL; - const char *attr = NULL; - const char *jcbstr = NULL; - - log_init ("flux-jstat"); - while ((ch = getopt_long (argc, argv, OPTIONS, longopts, NULL)) != -1) { - switch (ch) { - case 'h': /* --help */ - usage (0); - break; - case 'o': /* --testout */ - ofn = xasprintf ("%s", optarg); - break; - default: - usage (1); - break; - } - } - if (optind == argc) - usage (1); - - if (!(h = flux_open (NULL, 0))) - log_err_exit ("flux_open"); - - flux_log_set_appname (h, "jstat"); - cmd = argv[optind++]; - - if (!strcmp ("notify", cmd)) - rc = handle_notify_req (h, (const char *)ofn); - else if (!strcmp ("query", cmd) && optind == argc - 2) { - j = (const char *)(*(argv+optind)); - attr = (const char *)(*(argv+optind+1)); - rc = handle_query_req (h, strtol (j, NULL, 10), attr, ofn); - } - else if (!strcmp ("update", cmd) && optind == argc - 3) { - j = (const char *)(*(argv+optind)); - attr = (const char *)(*(argv+optind+1)); - jcbstr = (const char *)(*(argv+optind+2)); - rc = handle_update_req (h, strtol (j, NULL, 10), attr, jcbstr, ofn); - } - else - usage (1); - - flux_close (h); - log_fini (); - - return (!rc)? 0: 42; -} - -/* - * vi:tabstop=4 shiftwidth=4 expandtab - */ diff --git a/src/cmd/flux-submit b/src/cmd/flux-submit deleted file mode 100755 index 896024a5c1db..000000000000 --- a/src/cmd/flux-submit +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env lua - -------------------------------------------------------------- --- Copyright 2015 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - --- --- flux-submit: submit jobs using job.submit interface --- -local wreck = require 'wreck' .new () -if not wreck:parse_cmdline (arg) then - wreck:die ("Failed to process cmdline args\n") -end - --- If 'nokz' is used without --output, then store stdout and err in --- well-known kvs keys by default: -if wreck.job_options.nokz and not wreck:getopt('O') then - wreck.opts.O = "kvs://files.stdout" - if not wreck:getopt ('E') then wreck.opts.E = "kvs://files.stderr" end - - -- Regenerate wreck.output output file configuration: - wreck.output = wreck:output_file_config () -end - -local id, err = wreck:submit () -if not id then wreck:die ("%s\n", err) end -wreck:say ("Submitted jobid %d\n", id) -os.exit (0) --- vi: ts=4 sw=4 expandtab diff --git a/src/cmd/flux-wreck b/src/cmd/flux-wreck deleted file mode 100755 index f1ae9c05406b..000000000000 --- a/src/cmd/flux-wreck +++ /dev/null @@ -1,909 +0,0 @@ -#!/usr/bin/env lua - -------------------------------------------------------------- --- Copyright 2014 Lawrence Livermore National Security, LLC --- (c.f. AUTHORS, NOTICE.LLNS, COPYING) --- --- This file is part of the Flux resource manager framework. --- For details, see https://github.com/flux-framework. --- --- SPDX-License-Identifier: LGPL-3.0 -------------------------------------------------------------- - --- --- flux-wreck: simple wreck job utility --- -local f, err = require 'flux' .new() -local wreck = require 'wreck' -local hostlist = require 'flux.hostlist' -local posix = require 'flux.posix' -local prog = require 'flux.Subcommander'.create { - usage = "COMMAND [OPTIONS]", - handler = function (self, arg) - self:log ("COMMAND required\n") - self:help (1) - end -} - -local function printf (...) - io.stdout:write (string.format (...)) -end - -local function check_jobid_arg (self, id) - if not id then - io.stderr:write ("Error: Job id required\n") - self:help (1) - elseif not tonumber (id) then - self:die ("Invalid Job id '%s'\n", id) - end - return id -end - - -local function kvs_path (id, fmt, ...) - local id = assert (tonumber (id)) - local p, err = assert (wreck.id_to_path { flux = f, jobid = id }) - if fmt then - p = p .. "." .. string.format (fmt, ...) - end - return p -end - -local function dump_nokz_output (id) - local files = kvs_path (id, "files") - local stdout = f:kvs_get (files .. ".stdout") - local stderr = f:kvs_get (files .. ".stderr") - if stdout then io.stdout:write (stdout) end - if stderr then io.stderr:write (stderr) end -end - -local function attach_job_complete (state, taskio) - if not state then return false end - if state == "complete" or state == "reaped" then - if not taskio then return true end - return taskio:complete() - end - return false -end - -prog:SubCommand { - name = "attach", - description = "Attach to running or completed job output", - usage = "[OPTIONS] JOBID", - options = { - { name = "status", char = "s", - usage = "Include status after all output" }, - { name = "label-io", char = "l", - usage = "Label lines of output with task id" }, - { name = "no-follow", char = "n", - usage = "Don't wait for EOF on all output streams before exiting" }, - }, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local state = f:kvs_get (kvs_path (id, "state")) - local options = f:kvs_get (kvs_path (id, "options")) - local taskio - - if options.nokz and self.opt.l then - self:die ("--label-io not possible for jobs with 'nokz' option") - end - - if not options.nokz then - local err - taskio, err = wreck.ioattach { - flux = f, - jobid = id, - labelio = self.opt.l, - nofollow = self.opt.n, - on_completion = function () - if self.opt.n or attach_job_complete (state, taskio) then - f:reactor_stop() - end - end - } - if not taskio then - self:die ("Failed to connect to job %d: %s\n", id, err) - end - end - - local log, err = wreck.logstream { flux = f, jobid = id } - if not log then - self:die ("Failed to open logstream to job %d: %s\n", id, err) - end - - if state ~= "complete" and state ~= "reaped" then - local kz, err = f:kz_open (kvs_path (id, "input.files.stdin"), "w") - if kz then - local iow, err = f:iowatcher { - fd = posix.fileno (io.input()), - handler = function (iow, r) - if r.data then kz:write (r.data) end - if r.eof then kz:close () end - end - } - if not iow then self:die ("error opening stdin: %s\n", err) end - end - -- Watch state - local kw, err = f:kvswatcher { - key = kvs_path (id, "state"), - handler = function (kw, result) - if result then state = result end - if attach_job_complete (result, taskio) then - f:reactor_stop () - end - end - } - end - - if not attach_job_complete (state, taskio) then f:reactor () end - - if options.nokz then dump_nokz_output (id) end - log:dump() - if self.opt.s then - self.parent:run {"status", id} - end - os.exit (0) - end -} - -prog:SubCommand { - name = "dumplog", - description = "Dump error log stream for a wreck job", - usage = "[OPTIONS] JOBID", - options = nil, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local log, err = wreck.logstream { jobid = id, flux = f, oneshot = true } - if not log then - self:die ("Failed to open logstream to job %d: %s\n", id, err) - end - f:reactor() - end -} - -local function opt_sig (s) - if not s then return posix.SIGTERM end - if tonumber (s) then return tonumber (s) end - if not tonumber (posix[s]) then - prog:die ("Invalid signal '%s'\n", s) - end - return tonumber (posix [s]) -end - -prog:SubCommand { - name = "cancel", - description = "Cancel a pending job", - usage = "[-f|--force] JOBID", - options = { - { name = "force", char = "f", - usage = "Force cancel even if scheduler is not loaded" - }, - }, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local resp, err = f:rpc ("sched.cancel", { jobid = tonumber (id) }) - if not resp then - if err == "Function not implemented" and not self.opt.f then - prog:die ("job cancel not supported when scheduler not loaded") - elseif self.opt.f or err == "Invalid argument" then - prog:log ("Sending SIGKILL to %d\n", id) - local rc, err = f:sendevent ({signal = 9}, "wreck.%d.kill", id) - if not rc then self:die ("signal: %s\n", err) end - else - prog:die ("Unable to cancel %d: %s\n", id, err) - end - end - end -} - -prog:SubCommand { - name = "exclude", - description = "Exclude a node from running future jobs", - usage = "NODENAME", - options = { - { name = "kill", char = "k", - usage = "kill the jobs to which the node is allocated" - } - }, - handler = function (self, arg) - local function toboolean(X) - return not not X - end - - local nodename = arg[1] - if not nodename then - self:die ("Required argument NODENAME missing.") - end - local resp, err = f:rpc ("sched.exclude", { node = nodename, kill = toboolean (self.opt.k) }) - - if not resp then - if err == "Function not implemented" then - prog:die ("Node exclusion is not supported when scheduler not loaded") - else - prog:die ("Unable to exclude node %s: %s\n", nodename, err) - end - end - end -} - -prog:SubCommand { - name = "include", - description = "Include a node for scheduling", - usage = "NODENAME", - handler = function (self, arg) - local nodename = arg[1] - if not nodename then - self:die ("Required argument NODENAME missing.") - end - local resp, err = f:rpc ("sched.include", { node = nodename }) - if not resp then - if err == "Function not implemented" then - prog:die ("Node inclusion is not supported when scheduler not loaded") - else - prog:die ("Unable to include node %s: %s\n", nodename, err) - end - end - end -} - - -prog:SubCommand { - name = "kill", - description = "Kill a running job", - usage = "[OPTIONS] JOBID", - options = { - { name = "signal", char = "s", arg = "N", - usage = "Specify signal number N to be sent." } - }, - handler = function (self, arg) - local id = check_jobid_arg (self, arg[1]) - local lwj, err = f:kvsdir (kvs_path (id)) - if not lwj then self:die ("Job %d: %s\n", id, err) end - if lwj.state ~= "running" then - io.stderr:write ("Job "..id..": "..lwj.state.."\n") - os.exit (1) - end - local sig = opt_sig (self.opt.s) - local rc, err = f:sendevent ({signal = sig}, "wreck.%d.kill", id) - if not rc then self:die ("signal: %s\n", err) end - end -} - -prog:SubCommand { - name = "status", - description = "Return status of jobs", - usage = "JOBID", - handler = function (self, arg) - local jobid = check_jobid_arg (self, arg[1]) - local lwj, err = f:kvsdir (kvs_path (jobid)) - if not lwj then self:die ("Job %d: %s\n", jobid, err) end - print ("Job "..jobid.." status: "..lwj.state) - local code, msglist = wreck.status { flux = f, jobid = jobid } - if not code then self:die (msglist) end - for msg, hl in pairs (msglist) do - local s = string.format ("task%s: %s", tostring (hl:sort()), msg) - print (s) - end - os.exit (code) - end -} - - -local LWJ = {} -function LWJ.open (f, id, kvspath) - if not f or not id then - return nil, "Required arg missing" - end - if not kvspath then - kvspath = kvs_path (id) - end - local lwj, err = f:kvsdir (kvspath) - if not lwj then - return nil, "Failed to get lwj info: "..err - end - return setmetatable ({id = id, lwj = lwj }, LWJ) -end - -local function gettimeofday_as_table () - local ts, us = require 'flux.posix'.gettimeofday () - if not ts then return nil, us end - if type (ts) == "table" then return ts end - return { sec = ts, usec = us } -end - -local function now () - local t, err = gettimeofday_as_table () - if not t then error ("gettimeofday: "..err) end - return t.sec + (t.usec / (1000 * 1000)) -end - -function LWJ:timediff (tstart, tend, talt) - local t0 = self.lwj[tstart.."-time"] - if not t0 then return 0 end - local t1 = self.lwj[tend.."-time"] - if not t1 then t1 = self.lwj[talt.."-time"] end - if not t1 then t1 = now () end - local s = t1 - t0 - return s > 0 and s or 0 -end - -function LWJ:exit_string () - local flux = require 'flux' - local state = self.state - local exit_status = self.lwj.exit_status - if exit_status then - local max = exit_status.max - if max then - local s, code, core = flux.exitstatus (max) - state = s - if s == "exited" and code > 0 then - state = "failed" - end - end - end - return state -end - -function LWJ:state_string () - if self.state == "complete" then - return self:exit_string () - end - return self.state -end - -function LWJ:ranks () - local hl = hostlist.new() - local R = self.lwj.R_lite - if not R then - local rank = self.lwj.rank - if not rank then return nil end - for i in rank:keys() do hl:concat (i) end - else - for _,entry in ipairs (R) do - hl:concat (entry.rank) - end - end - return hl:sort() -end - -LWJ.__index = function (self, key) - if key == "state" then - if not self._state then - self._state = self.lwj.state - end - return self._state - elseif key == "ntasks" then - return self.lwj.ntasks - elseif key == "nnodes" then - return self.lwj.nnodes - elseif key == "runtime" then - return self:timediff ("starting", "complete", "failed") - elseif key == "t0" then - return self:timediff ("create", "starting", "failed") - elseif key == "t1" then - return self:timediff ("starting", "running", "failed") - elseif key == "t2" then - return self:timediff ("running", "complete", "failed") - elseif key == "start" then - return os.date ("%FT%T", self.lwj["starting-time"]) - elseif key == "end" then - return os.date ("%FT%T", self.lwj["complete-time"]) - elseif key == "created" then - return os.date ("%FT%T", self.lwj["create-time"]) - elseif key == "command" then - return self.lwj.cmdline[1] - elseif key == "name" then - return self.lwj.name - end - return LWJ [key] -end - -local function seconds_to_string (s) - local f = string.format - if s > (60*60*24) then - return f ("%.03fd", s / (60*60*24)) - elseif s > (60*60) then - return f ("%.03fh", s / (60*60)) - elseif s > 60 then - return f ("%.03fm", s / 60) - end - return f ("%.03fs", s) -end - -local function bracketify (args) - local r = {} - for _,v in ipairs (args) do - table.insert (r, "[" .. v .. "]") - end - return (r) -end - -local function joblist_from_args (arg) - local self = arg.self - local args = arg.args - - local max = tonumber (self.opt.n) or 25 - if max <= 0 then max = math.huge end - if #args == 0 then - return wreck.joblist{ flux = f, - max = max, - states = arg.states, - active_only = arg.active_only, - kvs_only = arg.kvs_only } - end - -- otherwise use dirs on cmdline - local hl,err = hostlist.union (unpack (bracketify (args))) - if not hl then return nil, err end - return wreck.jobids_to_kvspath { flux = f, jobids = hl:expand() } -end - -prog:SubCommand { - name = "ls", - usage = "[OPTIONS] [JOBIDs]", - options = { - { name = "max", char = 'n', arg="COUNT", - usage = "Display at most COUNT jobs", - }, - { name = "exclude-complete", char = 'x', - usage = "Skip the listing of complete jobs in the kvs" - } - }, - description = "List jobs in kvs", - handler = function (self, arg) - local dirs,err = joblist_from_args { self = self, args = arg, - active_only = self.opt.x } - if not dirs then self:die (err) end - if #dirs == 0 then return end - local fmt = "%6s %6s %-9s %20s %12s %8s %-.13s\n"; - printf (fmt, "ID", "NTASKS", "STATE", "START", - "RUNTIME", "RANKS", "COMMAND/NAME") - for _,dir in pairs (dirs) do - local id = dir:match ("(%d+)$") - if tonumber (id) then - local j, err = LWJ.open (f, id, dir) - if j then - printf (fmt, id, j.ntasks, j:state_string(), j.start, - seconds_to_string (j.runtime), - tostring (j:ranks()), - j.name or j.command:match ("([^/]+)$")) - end - end - end - end -} - -prog:SubCommand { - name = "uri", - usage = "[OPTIONS] [JOBIDs]", - options = { - { name = "max", char = 'n', arg="COUNT", - usage = "Display at most COUNT jobs", - }, - { name = "bare", char = 'b', - usage = "Dispaly only the URI", - } - }, - description = "List FLUX_URI for jobs that are Flux instances", - handler = function (self, arg) - -- Only include running jobs: - local states = { include = { running = true } } - local dirs,err = joblist_from_args { self = self, - args = arg, - states = states, - active_only = true } - if not dirs then self:die (err) end - if #dirs == 0 then return end - local fmt = "%6s %6s %-9s %-40s %-.13s\n"; - if self.opt.b then - if #dirs > 1 then self:die ("--bare only works with one job") end - else - printf (fmt, "ID", "NTASKS", "STATE", "FLUX_URI", "COMMAND") - end - for _,dir in pairs (dirs) do - local id = dir:match ("(%d+)$") - if tonumber (id) then - local j, err = LWJ.open (f, id, dir) - if not j then self:die ("job%d: %s", id, err) end - local uri, err = f:kvs_get (kvs_path (id, "flux.remote_uri")) - if self.opt.b then - if err then self:die ("job%d: not a Flux instance", id) end - printf ("%s\n", uri) - else - if err then uri = "" end - printf (fmt, id, j.ntasks, j:state_string(), uri, - j.command:match ("([^/]+)$")) - end - end - end - end -} - -prog:SubCommand { - name = "timing", - usage = "[OPTIONS] [JOBIDs]...", - options = { - { name = "max", char = 'n', arg="COUNT", - usage = "Display at most COUNT jobs", - } - }, - description = "List timings of jobs in kvs", - handler = function (self, arg) - local dirs,err = joblist_from_args {self = self, args = arg} - if not dirs then self:die (err) end - if #dirs == 0 then return end - local fmt = "%6s %12s %12s %12s %12s %12s\n" - printf (fmt, "ID", "NTASKS", "STARTING", "RUNNING", "COMPLETE", "TOTAL") - for _,dir in pairs (dirs) do - local id = dir:match ("(%d+)$") - if tonumber (id) then - local j, err = LWJ.open (f, id, dir) - if not j then self:die ("job%d: %s", id, err) end - printf (fmt, id, - j.ntasks, - seconds_to_string (j.t0), - seconds_to_string (j.t1), - seconds_to_string (j.t2), - seconds_to_string (j.runtime)) - end - end - end -} - -prog:SubCommand { - name = "kvs-path", - usage = "JOBID", - description = "Display path in KVS to job ID", - handler = function (self, arg) - if not arg[1] then self:die ("Job ID argument required\n") end - print (kvs_path (arg[1])) - end -} - -prog:SubCommand { - name = "last-jobid", - description = "Display the last jobid in lwj id sequence", - usage = "[OPTIONS]", - options = { - { name = "kvs-path", char = 'p', - usage = "Return path to job in kvs instead of jobid", - } - }, - handler = function (self, arg) - local req = { - name = "lwj", - preincrement = 0, - postincrement = 0, - create = false - } - local resp, err = f:rpc ("seq.fetch", req) - if not resp then - self:die ("No last jobid found: %s", err) - end - local result = resp.value - if self.opt.p then - result = kvs_path (result) - end - print (result) - end -} - -prog:SubCommand { - name = "setenv", - usage = "[VAR=VALUE|all]", - description = "Export environment to all jobs", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local getenv = wreck.get_filtered_env - local env = f:kvs_get ("lwj.environ") or {} - for _,expr in ipairs (arg) do - if expr == "all" then - for k,v in pairs(getenv()) do - env[k] = v - end - else - local var,val = expr:match("(.*)=(.*)") - env[var] = val - end - end - f:kvs_put ("lwj.environ", env) - f:kvs_commit () - end -} - -prog:SubCommand { - name = "unsetenv", - usage = "VAR [VAR...]", - description = "Export environment to all jobs", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local env = f:kvs_get ("lwj.environ") or {} - for _,var in ipairs (arg) do - env[var] = nil - end - f:kvs_put ("lwj.environ", env) - f:kvs_commit () - end -} - -prog:SubCommand { - name = "getenv", - usage = "VAR [VAR...]", - description = "Export environment to all jobs", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local env = f:kvs_get ("lwj.environ") or {} - if #arg == 0 then - for k,v in pairs (env) do - print (k.."="..v) - end - end - for _,k in ipairs (arg) do - local v = env[k] or "" - print (k.."="..v) - end - end -} - -prog:SubCommand { - name = "setopt", - usage = "option[=value]", - description = "Set global WRECK job options", - handler = function (self, arg) - if #arg == 0 then - self:die ("Missing option!") - end - local f = assert (require 'flux'.new ()) - local options = f:kvs_get ("lwj.options") or {} - for _,s in pairs (arg) do - local opt,val = s:match("([^=]+)=?(.*)") - if not val or val == "" then - options[opt] = true - elseif val == "false" or val == "no" or val == 0 then - options[opt] = nil - else - options[opt] = tonumber(val) or val - end - end - f:kvs_put ("lwj.options", options) - f:kvs_commit () - end -} - -prog:SubCommand { - name = "getopt", - usage = "[option]", - description = "Print global WRECK options", - handler = function (self, arg) - local f = assert (require 'flux'.new ()) - local options = f:kvs_get ("lwj.options") or {} - if #arg == 0 then - for k,v in pairs (options) do - print (k.."="..tostring (v)) - end - else - for _,k in pairs (arg) do - v = options[k] - if v then - print (k.."="..tostring (v)) - else - print (k.."=false") - end - end - end - end -} - --- return keys in dir as a table sorted by number -local function sorted_keys (dir) - local results = {} - for k in dir:keys () do - table.insert (results, k) - end - table.sort (results, function (a,b) return tonumber (a) < tonumber (b) end) - return results -end - -local function remove_if_empty (key, r) - local results = r or {} - local dir = f:kvsdir (key) - if not dir or dir.state then return false, results end - local remove = true - for k in dir:keys () do - remove, results = remove_if_empty (key .. "." .. k, results) - end - if remove then - f:kvs_unlink (key) - table.insert (results, key) - end - return remove, results -end - -prog:SubCommand { - name = "purge", - usage = "[OPTIONS]", - options = { - { name = "verbose", char = "v", - usage = "Increase verbosity." }, - { name = "target-size", char = 't', arg = "N", - usage = "Specify a target number of lwj entries to keep" - }, - { name = "remove", char = "R", - usage = "Remove all eligible lwj entries." }, - { name = "max", char = "m", arg = "N", - usage = "Maximum number of entries to target." }, - { name = "keep", char = "k", arg = "CODE", - usage = "Keep entries where CODE is true" }, - }, - description = "Purge old jobs entries from kvs", - handler = function (self, arg) - local tt = require 'flux.timer'. new() - local verbose = self.opt.v - - --- - -- Gather LWJ path list - -- Only include complete jobs - local states = { include = { complete = true, - failed = true - } - } - local dirs = wreck.joblist{ flux = f, - kvs_only = true, - states = states - } or {} - if verbose then - self:log ("%4.03fs: got lwj list (%d entries)\n", tt:get0(), #dirs) - end - - local total = #dirs - local target = tonumber (self.opt.t) - local max = tonumber (self.opt.m) or math.huge - local keep = function (id) return false end - if self.opt.k then - local err - keep, err = - loadstring ('local id = tonumber (...); return '..self.opt.k) - if not keep then - self:die ("Failed to compile --keep option: %s\n", err) - end - end - - -- Without --target-size option, just list current number of lwj entries - if not target then - self:log ("%d total job entries\n", total) - elseif total <= target then - -- If current total is less than target, then log and exit - self:log ("%d total jobs at or below target size of %d\n", - total, target) - else - -- Otherwise, we need to purge some entries: - -- - local n = math.min (total - target, max) - - -- Remove the first n ids using lwj-completed directory - local r = {} - (function() -- anonymous function used for early return - local completed = f:kvsdir ("lwj-complete") - for _,hb in ipairs (sorted_keys (completed)) do - local hb_unlink = true - local d = completed [hb] - for _,id in ipairs (sorted_keys (d)) do - -- Save name of this kvs path: - local complink = tostring (d).."."..id - if keep (id) then - hb_unlink = false - else - table.insert (r, id) - if self.opt.R then - local t, target = f:kvs_type (complink) - if t == "symlink" then - f:kvs_unlink (target) - end - f:kvs_unlink (complink) - end - n = n - 1 - if n == 0 then return end - end - end - -- - -- Finished iterating all entries in this hb subdir, - -- should be ok to remove the hb dir now: - if self.opt.R and hb_unlink then - f:kvs_unlink ("lwj-complete."..hb) - end - end - end)() - - -- gather ids to remove in hostlist for condensed output: - -- - local hl = require 'flux.hostlist' .new (table.concat (r, ",")):uniq () - local rmdirs = {} - if self.opt.R then - f:kvs_commit() - if verbose then - self:log ("%4.03fs: unlinked %d entries\n", tt:get0(), #hl) - end - _, rmdirs = remove_if_empty ("lwj") - f:kvs_commit () - elseif verbose then - self:log ("%4.03fs: finished walking %d entries in lwj-complete\n", - tt:get0(), #hl) - end - - local idstring = "" - if #hl > 0 then - idstring = ": "..tostring (hl):match ("%[?([^]]+)%]?") - end - self:log ("%s %d lwj entries%s\n", - self.opt.R and "removed" or "would remove", #hl, - idstring) - if #rmdirs > 0 then - self:log ("removed %d empty dirs under lwj.\n", #rmdirs) - if verbose then - self:log ("removed: %s\n", table.concat (rmdirs, ", ")) - end - end - if verbose then - self:log ("%4.03fs: all done.\n", tt:get0()); - end - end - end -} - -prog:SubCommand { - name = "sched-params", - description = "Set/Get scheduler parameters at runtime.", - usage = "get or set ITEM=VAL", - - handler = function (self, arg) - local action = arg[1] - if action == "set" then - local params = arg[2] - if not params then - self:die ("No arguments specified for setting scheduler parameters.") - end - - local resp, err = f:rpc ("sched.params.set", { param = tostring(params) }) - if not resp then - if err == "Function not implemented" then - prog:die ("Scheduler parameters cannot be updated when scheduler is not loaded") - else - prog:die ("Unable to set scheduler parameters. %s\n", err) - end - end - else -- Else corresponding to the 'get' action, return all current values - local params = arg[2] - local resp, err = f:rpc ("sched.params.get", { }) - if not resp then - if err == "Function not implemented" then - prog:die ("Scheduler parameters cannot be queried when scheduler is not loaded") - else - prog:die ("Unable to obtain scheduler parameters. %s\n", err) - end - end - - for k,v in pairs(resp) do - if (k == "delay-sched") then -- Return a simple integer, but print true/false for consistency - local v_str = "" - if (v == 1) then - v = "true" - else - v = "false" - end - end - if not params then - print(k.."="..v) - elseif string.find(params,k,1,true) then -- We have strings with '-', so search for plain strings - print(k.."="..v) - end - end - end - end -} - --- Check for valid connection to flux: -if not f then prog:die ("Connecting to flux failed: %s\n", err) end - --- Process cmdline, run any subcommand, or exit with failure: -prog:run (arg) --- vi: ts=4 sw=4 expandtab - diff --git a/src/cmd/flux-wreckrun b/src/cmd/flux-wreckrun deleted file mode 100755 index d53829a49faa..000000000000 --- a/src/cmd/flux-wreckrun +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/env lua - -------------------------------------------------------------------------------- --- Modules: -------------------------------------------------------------------------------- -local flux = require 'flux' -local posix = require 'flux.posix' -local timer = require 'flux.timer' - -local prog = string.match (arg[0], "([^/]+)$") -local shortprog = prog:match ("flux%-(.+)$") - --- --- Termination state needs to remain a global for access from --- signal handler functions. See setup_signal_handlers() below. --- -terminate = false - -------------------------------------------------------------------------------- --- Local functions: -------------------------------------------------------------------------------- --- --- Check that parameter [a] is an integer -- -local function is_integer (a) - local b = tonumber (a) - return (type (b) == "number") and (math.floor(b) == b) -end - ---- --- Get the LWJ return code as highest of all task return codes ---- -local function lwj_return_code (f, wreck, id) - local max, msgs = wreck.status { flux = f, jobid = id } - local fn = getmetatable (wreck) [ max > 0 and "say" or "verbose" ] - for s,h in pairs (msgs) do - fn (wreck, "task%s %s: %s\n", #h > 1 and "s" or "", tostring (h:sort()), s) - end - return max -end - --- --- Helper for fake_R_lite(): generate a fake range string from 0 .. n, --- when n >= 1 --- -local function range_string (n) - if not n or n == 0 then return nil end - local s = "0" - if n > 1 then - s = s .. "-" .. n-1 - end - return s -end - --- --- Generate a fake R_lite representation for the current wreck request. --- If -g, --gpus-per-task was used on the command line, include GPUs. --- --- This assists with testing without sched or with the -I, --immediate flag. --- -local function fake_R_lite (f, wreck) - local ncores, ngpus = wreck.ncores, wreck.ngpus - local R_lite = {} - local nnodes = wreck.nnodes > 0 and wreck.nnodes or f.size - local cores_per_node = math.ceil (ncores / nnodes) - local gpus_per_node = ngpus and math.ceil (ngpus / nnodes) - - -- Allocate cores (and gpus) to R_lite in rank order until - -- no more cores are required - -- - for rank = 0, nnodes - 1 do - local corecnt = math.min (ncores, cores_per_node) - local gpucnt = ngpus and math.min (wreck.ngpus, gpus_per_node) - table.insert (R_lite, { rank = rank, - children = { - core = range_string (corecnt), - gpu = range_string (gpucnt) } - }) - ncores = ncores - corecnt - if ngpus then - ngpus = ngpus - gpucnt - end - if ncores == 0 then - return R_lite - end - end - return R_lite -end - -local function wreckrun_hooks_compile (wreck) - local code = wreck:getopt ("P") - local pre_launch_fn = nil - if code then - wreck:verbose ("compiling code '%s'\n", code) - pre_launch_fn, err = loadstring ("local wreck, lwj = ...; " .. code) - if not pre_launch_fn then - wreck:die ("--pre-launch-hook: %s\n", err) - end - end - -- Add a hooks table to existing 'wreck' object - wreck.hooks = { pre_launch = pre_launch_fn } - return true -end - -local function wreckrun_pre_launch ( wreck, lwj) - if wreck.hooks.pre_launch then - local r, v = pcall (wreck.hooks.pre_launch, wreck, lwj) - if not r then return nil, v end - end - return true -end - -------------------------------------------------------------------------------- --- Main program: -------------------------------------------------------------------------------- --- Open connection to flux broker: - -local f, err = flux.new () -if not f then - io.stderr:write ("Unable to connect to flux broker: "..err) - os.exit (1) -end - --- Parse cmdline args: --- -local wreck = require 'wreck' .new { prog = shortprog, flux = f } -local ioattach = require 'wreck'.ioattach -local logstream = require 'wreck'.logstream -local taskio -local log -local sigtimer -local state = "" - -wreck:add_options ({ - { name = 'pre-launch-hook', char = "P", arg = "CODE", - usage = "execute Lua CODE before launch." }, - { name = 'detach', char = "d", - usage = "Detach immediately after starting job" }, - { name = 'wait-until', char = "w", arg = "STATE", - usage = "Do not process stdio, but block until 'STATE'" }, - { name = 'immediate', char = "I", - usage = "Bypass scheduler and run immediately" }, -}) - -if not wreck:parse_cmdline (arg) then - wreck:die ("Failed to process cmdline args\n") -end - --- if nokz is in effect and the --output option is not used, AND --- either --detach or --wait-until are active, then store output by --- default in the kvs; o/w it is just dropped: -if (wreck.job_options.nokz and not wreck:getopt ('O')) and - (wreck:getopt ('d') or wreck:getopt ('w')) then - wreck.opts.O = "kvs://files.stdout" - if not wreck:getopt ('E') then wreck.opts.E = "kvs://files.stderr" end - - -- Regenerate wreck.output output file configuration: - wreck.output = wreck:output_file_config () -end - -wreckrun_hooks_compile (wreck) - --- Start in-program timer: -local tt = timer.new() - --- --- Create a new job in kvs --- -local jobid, err = wreck:createjob () -if not jobid then wreck:die ("%s\n", err) end - -wreck:verbose ("%4.03fs: Registered jobid %d\n", tt:get0(), jobid) - --- --- Get a handle to this lwj kvsdir: --- -local lwj, err = wreck:kvsdir (jobid) -if not lwj then wreck:die ("f:kvsdir(lwj.%d): %s\n", jobid, err) end - --- --- Check if job state is "complete" and all IO from tasks have closed: --- -local function check_job_completed () - if state == "failed" then - log:dump() - wreck:die ("job %d failed\n", jobid) - end - if (not taskio or taskio:complete()) and - (state == "completing" or state == "complete" or state == "reaped") then - local rc = lwj_return_code (f, wreck, jobid) - if rc == 0 then - wreck:verbose ("%.3fs: All tasks completed successfully.\n", - tt:get0 ()); - end - os.exit (rc) - end -end - ---- ---- Attach stdio kz streams and start all watchers ---- -local function wreck_attach_stdio (wreck) - -- - -- Open output to all tasks: - -- - taskio, err = ioattach { - flux = f, - jobid = jobid, - ntasks = wreck.ntasks, - labelio = wreck.opts.l, - nokz = wreck.job_options.nokz, - ioservices = wreck.ioservices, - on_completion = check_job_completed - } - if not taskio then wreck:die ("error opening task io: %s\n", err) end - - -- - -- Open stdin - -- - local kz, err = f:kz_open (tostring (lwj)..".input.files.stdin", "w") - if not kz then wreck:die (err) end - - -- - -- Add a file descriptor iowatcher for this script's stdin: - -- - local iow, err = f:iowatcher { - fd = posix.fileno (io.input()), - handler = function (iow, data) - if data.data then kz:write (data.data) end - if data.eof then kz:close () end - end - } - if not iow then wreck:die ("error opening stdin: %s\n", err) end - - return taskio -end - - --- --- Install watcher for this job's state before issuing any run request, --- (So that no state events are missed) --- -local kw, err = f:kvswatcher { - key = string.format ("%s.state", wreck:lwj_path (jobid)), - handler = function (kw, result) - if result then - state = result - wreck:verbose ("%-4.03fs: State = %s\n", tt:get0(), result) - if wreck.opts.w and state == wreck.opts.w then - os.exit (0) - end - check_job_completed () - end - end -} -if not kw then wreck:die ("kvs watch: %s\n", err) end - -local submitted = false -if not wreck:getopt ("I") then - -- Attempt to submit this existing job via submit-nocreate RPC: - -- - local rc, err = f:rpc ("job.submit-nocreate", { jobid = jobid }) - if rc then - submitted = true - wreck:verbose ("%-4.03fs: job.submit: Success\n", tt:get0()); - else - wreck:verbose ("%-4.03fs: job.submit: %s\n", tt:get0(), err) - end -end - -if not submitted then - -- - -- If submit failed due to lack of scheduler or use of the - -- -I, --immediate option, manually create R_lite and - --- send event to run the job. - -- - R_lite = fake_R_lite (f, wreck); - if not R_lite then wreck:die ("Failed to generate R_lite\n") end - lwj.R_lite = R_lite - lwj:commit() - - -- Ensure lwj nnodes matches fake allocation - lwj.nnodes = tonumber (wreck.nnodes) - lwj.ntasks = tonumber (wreck.ntasks) - - -- Execute any pre-launch hooks - local r, err = wreckrun_pre_launch (wreck, lwj) - if not r then - lwj.state = "failed" - lwj:commit() - wreck:die ("pre-launch-hook failed: %s", err) - end - - lwj:commit () - - -- Now send run event: - wreck:verbose ("%-4.03fs: Sending run event\n", tt:get0()) - local rc,err = f:sendevent ("wrexec.run.%d", jobid) - if not rc then wreck:die ("sendevent: %s\n", err) end - if wreck.opts.d then - print (jobid) - os.exit(0) - end -end - --- Only process stdio if no --wait-until option used -if not wreck.opts.w then - taskio = wreck_attach_stdio (wreck) -end - -log, err = logstream { - flux = f, - jobid = jobid, -} -if not log then wreck:die ("error opening log stream: %s\n", err) end - -local s, err = f:sighandler { - sigmask = { posix.SIGINT, posix.SIGTERM }, - handler = function (f, s, sig) - terminate = true - f:reactor_stop () - end -} - -local s, err = f:sighandler { - sigmask = { posix.SIGALRM }, - handler = function (f, s, sig) - wreck:say ("Killed by SIGALRM: state = %s\n", state) - os.exit (128+posix.SIGALRM) - end -} - --- --- Begin reactor loop: --- -local sigtimer = nil - -repeat - local r, err = f:reactor() - if not r then wreck:die ("flux_reactor: %s", err) end - -- - -- If we catch a signal then lwj:watch() will be interrupted. - -- Check to see if we should terminate the job now: - -- - if terminate then - wreck:verbose ("%4.03fs: Killing LWJ %d\n", tt:get0(), jobid) - local rc,err = f:sendevent ("wreck.%d.kill", jobid) - if not rc then - wreck:say ("Error: Failed to send kill event: %s", err) - end - if not sigtimer then - sigtimer = timer.new() - sigtimer:get() - elseif sigtimer:get() < 1.0 then - wreck:say ("Detaching from job. Processes may still be running\n"); - os.exit (0); - end - terminate = false - end -until false - - --- vi: ts=4 sw=4 expandtab diff --git a/src/cmd/flux.c b/src/cmd/flux.c index 0160e6044cd1..50c2622db733 100644 --- a/src/cmd/flux.c +++ b/src/cmd/flux.c @@ -194,11 +194,6 @@ int main (int argc, char *argv[]) flux_conf_get ("rc3_path", flags), 0); environment_from_env (env, "FLUX_PMI_LIBRARY_PATH", flux_conf_get ("pmi_library_path", flags), 0); - environment_from_env (env, "FLUX_WRECK_LUA_PATTERN", - flux_conf_get ("wreck_lua_pattern", flags), 0); - environment_from_env (env, "FLUX_WREXECD_PATH", - flux_conf_get ("wrexecd_path", flags), 0); - if ((flags & CONF_FLAG_INTREE)) environment_push (env, "FLUX_CONF_INTREE", "1"); diff --git a/src/common/Makefile.am b/src/common/Makefile.am index 316943e5bab2..058bcb44b18f 100644 --- a/src/common/Makefile.am +++ b/src/common/Makefile.am @@ -5,7 +5,6 @@ SUBDIRS = libtap \ libutil \ libflux \ libkvs \ - libjsc \ libjob \ libzio \ libcompat \ @@ -44,7 +43,6 @@ libflux_core_la_SOURCES = libflux_core_la_LIBADD = \ $(builddir)/libflux/libflux.la \ $(builddir)/libkvs/libkvs.la \ - $(builddir)/libjsc/libjsc.la \ $(builddir)/libjob/libjob.la \ $(builddir)/libsubprocess/libsubprocess.la \ libflux-internal.la diff --git a/src/common/core.h b/src/common/core.h index 6b671f3c0276..2293fb355dc0 100644 --- a/src/common/core.h +++ b/src/common/core.h @@ -17,7 +17,6 @@ #include "core/flux.h" #include "core/kvs.h" -#include "core/jstatctl.h" #include "core/job.h" #endif /* !_FLUX_CORE_H */ diff --git a/src/common/libflux-core.map b/src/common/libflux-core.map index c34072d193e0..78eddb1a1ada 100644 --- a/src/common/libflux-core.map +++ b/src/common/libflux-core.map @@ -1,6 +1,5 @@ { global: flux_*; - jsc_*; JSC_*; local: *; }; diff --git a/src/common/libflux/Makefile.am b/src/common/libflux/Makefile.am index 44848d4d8365..c735f5056f9d 100644 --- a/src/common/libflux/Makefile.am +++ b/src/common/libflux/Makefile.am @@ -24,8 +24,6 @@ installed_conf_cppflags = \ -DINSTALLED_PMI_LIBRARY_PATH=\"$(fluxlibdir)/libpmi.so\" \ -DINSTALLED_RC1_PATH=\"$(fluxrcdir)/rc1\" \ -DINSTALLED_RC3_PATH=\"$(fluxrcdir)/rc3\" \ - -DINSTALLED_WRECK_LUA_PATTERN=\"$(sysconfdir)/wreck/lua.d/*.lua\" \ - -DINSTALLED_WREXECD_PATH=\"$(fluxlibexecdir)/wrexecd\" \ -DINSTALLED_CMDHELP_PATTERN=\"${datadir}/flux/help.d/*.json\" \ -DINSTALLED_NO_DOCS_PATH=\"${datadir}/flux/.nodocs\" \ -DINSTALLED_RUNDIR=\"${runstatedir}/flux\" \ @@ -45,8 +43,6 @@ intree_conf_cppflags = \ -DINTREE_PMI_LIBRARY_PATH=\"$(abs_top_builddir)/src/common/.libs/libpmi.so\" \ -DINTREE_RC1_PATH=\"$(abs_top_srcdir)/etc/rc1\" \ -DINTREE_RC3_PATH=\"$(abs_top_srcdir)/etc/rc3\" \ - -DINTREE_WRECK_LUA_PATTERN=\"$(abs_top_srcdir)/src/modules/wreck/lua.d/*.lua\" \ - -DINTREE_WREXECD_PATH=\"$(abs_top_builddir)/src/modules/wreck/wrexecd\" \ -DINTREE_CMDHELP_PATTERN=\"${abs_top_builddir}/etc/flux/help.d/*.json\" \ -DINTREE_KEYDIR=\"${abs_top_builddir}/etc/flux\" \ -DINTREE_NO_DOCS_PATH=\"${abs_top_builddir}/etc/flux/.nodocs\" \ diff --git a/src/common/libflux/conf.c b/src/common/libflux/conf.c index 1cfc25d9ba2b..8ed3bedaf8b0 100644 --- a/src/common/libflux/conf.c +++ b/src/common/libflux/conf.c @@ -34,10 +34,6 @@ static struct config default_config[] = { { "cmdhelp_pattern",INSTALLED_CMDHELP_PATTERN, INTREE_CMDHELP_PATTERN }, { "pmi_library_path", INSTALLED_PMI_LIBRARY_PATH, INTREE_PMI_LIBRARY_PATH }, - { "wrexecd_path", INSTALLED_WREXECD_PATH, INTREE_WREXECD_PATH }, - { "wreck_lua_pattern", - INSTALLED_WRECK_LUA_PATTERN, - INTREE_WRECK_LUA_PATTERN }, { "keydir", NULL, INTREE_KEYDIR }, { "no_docs_path", INSTALLED_NO_DOCS_PATH, INTREE_NO_DOCS_PATH }, { "rundir", INSTALLED_RUNDIR, NULL }, diff --git a/src/common/libjsc/Makefile.am b/src/common/libjsc/Makefile.am deleted file mode 100644 index f786958e6e58..000000000000 --- a/src/common/libjsc/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -AM_CFLAGS = \ - $(WARNING_CFLAGS) \ - $(CODE_COVERAGE_CFLAGS) - -AM_LDFLAGS = \ - $(CODE_COVERAGE_LIBS) - -AM_CPPFLAGS = \ - -I$(top_srcdir) \ - -I$(top_srcdir)/src/include \ - -I$(top_builddir)/src/common/libflux \ - $(ZMQ_CFLAGS) $(JANSSON_CFLAGS) - -noinst_LTLIBRARIES = libjsc.la -fluxcoreinclude_HEADERS = jstatctl.h - -libjsc_la_SOURCES = \ - jstatctl.c diff --git a/src/common/libjsc/README.md b/src/common/libjsc/README.md deleted file mode 100644 index 0c5799ca0b39..000000000000 --- a/src/common/libjsc/README.md +++ /dev/null @@ -1,180 +0,0 @@ -Job Status and Control Application Programming Interface -=================== - -The Job Status and Control (JSC) API is a high-level abstraction that -allows its client software to monitor and control the status of Flux -jobs. It is designed to expose the job status and control abstraction -in a way to hide the underlying data layout of job information stored -within Flux's KVS data store. We expect that schedulers and runtime -tools will be its main users. This abstraction provides the producers -of job information including a task and program execution service -module such as `flux-wreckrun` with an opportunity to change and -optimize the data layout of jobs within the KVS without presenting -major impacts to the implementation of the schedulers and runtime tools. - -1. Design Consideration -------------- -The main design considerations are the following: ->1. Abstract out implementation-dependent job states; ->2. Provide flexible and easily extensible mechanisms to pass job -information; ->3. Use a minimalistic API set. - -The first consideration has led us to use a C enumerator (i.e., -*job\_state\_t*) to capture the job states. However, because Flux has -not yet defined its job schema, the second consideration discouraged us -to use a C user-defined type to pass job information with the client -software. Instead, JSC uses an JSON to capture the job information and -introduce the notion of Job Control Block (JCB) to have a structure on -this information. We will try to keep backward compatibility on JCB's -structure as we will extend this to keep abreast of the evolution of -Flux's job schema. Finally, the third consideration has led us to -introduce three simple API calls as the main JSC idioms: job -status-change notification as well as JCB query and update. Client -software can use the notification call to get the status of a job -asynchronously on a state change; the query and update calls to fetch -and update a job's JCB, respectively. - -2. Job States -------------- -The JSC API converts the state strings produced by `flux-wreckrun` and -the scheduler framework service comms module into a C enumerator: -*job\_state\_t*. Its elements are shown in Table 2-1. If the raw state -strings are changed in the future, one must accommodate this JCB -implementation accordingly--mapping the new strings to these state -elements. We expect the state set will further be extended as we will -add more advanced services such as elastic job and resource management -service. - -| Elements | Comment | -|--------------|------------------------------------------------------| -| J_NULL | No state has been assigned | -| J_RESERVED | Reserved in KVS | -| J_SUBMITTED | Added to KVS | -| J_PENDING | Pending | -| J_SCHEDREQ | Resource selection requested | -| J_ALLOCATED | Resource allocated/contained in the Flux instance | -| J_RUNREQUEST | Requested to be executed | -| J_STARTING | Starting | -| J_STOPPED | Stopped | -| J_RUNNING | Running | -| J_CANCELLED | Cancelled | -| J_COMPLETE | Complete | -| J_REAPED | Reaped to the upper-level Flux instance | -| J_FAILED | Failed before running | -| J_FOR\_RENT | To be extended | - -**Table 2-1** Job state elements - - -| Key | Macro | Value Type | Comment | -|------------|----------------|----------------|--------------------------------------------------------------------------------| -| jobid | JSC_JOBID | 64-bit integer | Job id | -| state-pair | JSC_STATE\_PAIR| dictionary | A dictionary containing this old and new states of the job. See Table 3-2. | -| rdesc | JSC_RDESC | dictionary | Information on the resources owned by this job. See Table 3-3. | -| rdl | JSC_RDL | string | RDL binary string allocated to the job | -| R_lite | JSC_R\_LITE | string | R\_lite serialized JSON string allocated to the job | - -**Table 3-1** Keys and values of top-level JCB attributes - - -| Key | Macro | Value Type | Comment | -|------------|-------------------------|----------------|---------------------------------------| -| ostate | JSC_STATE\_PAIR\_OSTATE | 64-bit integer | Old state (a *job\_state\_t* element) | -| nstate | JSC_STATE\_PAIR\_NSTATE | 64-bit integer | New state (a *job\_state\_t* element) | - -**Table 3-2** Keys and values of *state-pair* attribute - - -| Key | Macro | Value Type | Comment | -|------------|------------------|-----------------|---------------| -| nnodes | JSC_RDESC\_NNODES | 64-bit integer | Node count | -| ntasks | JSC_RDESC\_NTASKS | 64-bit integer | Process count | -| ncores | JSC_RDESC\_NCORES | 64-bit integer | core count | -| ngpus | JSC_RDESC\_NGPUS | 64-bit integer | GPU count | -| walltime | JSC_RDESC\_WALLTIME | 64-bit integer | Walltime | - -**Table 3-3** Keys and values of *rdesc* attribute - - -3. Job Control Block -------------- -Job Control Block (JCB) is our data schema containing the information -needed to manage a particular job. It contains information such as -jobid, resources owned by the job, as well as the processes spawned by -the job. The JSC API converts the raw information on a job into an -JCB, implemented as an JSON dictionary object. Our current JCB -structure is shown in Table 3-1 through 3-6. As Flux's job schema -evolves, we will extend JCB while trying our best to keep backward -compatibility. - - -4. Application Programming Interface -------------- -The Job Status and Control API currently provides three main calls: - -> Note: typedef int (\*jsc\_handler\_f)(json\_object \*base\_jcb, void -\*arg, int -errnum); -> ->- int jsc\_notify\_status (flux\_t h, jsc\_handler\_f callback, void -\*d); ->- int jsc\_query\_jcb (flux\_t h, int64\_t jobid, const char \*key, -char -\*\*jcb); ->- int jsc\_update\_jcb (flux\_t h, int64\_t jobid, const char \*key, -const char -\*jcb); - - - -#### jsc\_notify\_status -Register a *callback* to the asynchronous status change notification -service. *callback* will be invoked when the state of a job changes. -The *jobid* and *state-pair* as shown in Table 3-2 will be passed as -*base_jcb* into the *callback*. *d* is arbitrary data that will -transparently be passed into *callback*. However, one should pass its -*flux_t* object as part of this callback data. Note that the caller -must start its reactor to get an asynchronous status change -notification via *callback*. This is because it uses the KVS-watch -facility which has the same limitation. One can register multiple -callbacks by calling this function multiple times. The callbacks will -be invoked in the order they are registered. Returns 0 on success; -otherwise -1. - - -#### jsc\_query\_jcb -Query the *key* attribute of JCB of *jobid*. The JCB info on this -attribute will be passed via *jcb*. It is the responsibility of the -caller to release *jcb*. All ownership associated with the -sub-attributes in *jcb*'s hierarchy are transferred to *jcb*, so that -json_object_put (\*jcb) will free this hierarchy in its entirety. -Returns 0 on success; otherwise -1. - -#### jsc\_update\_jcb -Update the *key* attribute within the JCB of *jobid*. The top-level -attribute of *jcb* should be the same as *key*. Returns 0 on success; -otherwise -1. This will not release *jcb* so it is the responsibility -of the caller to free *jcb*. - ->**Notes:** - ->1. JCB granularity optimization -- one can optimize the amounts of JCB -information piggybacked with each notification (*base_jcb*). One can -further extend the single attribute-wise query/update pattern to -group-wise ones once the access patterns of JCS API's clients are -known. - -5. Testing -------------- - -To facilitate the testing of this API, we created an utility command: -`flux-jstat`. Its usage is the following: - - Usage: flux-jstat notify - flux-jstat query jobid - flux-jstat update jobid - -Further, `flux-core/t/t2001-jsc.t` contains various test cases that use -this utility. - diff --git a/src/common/libjsc/jstatctl.c b/src/common/libjsc/jstatctl.c deleted file mode 100644 index 21fa98881d42..000000000000 --- a/src/common/libjsc/jstatctl.c +++ /dev/null @@ -1,862 +0,0 @@ -/************************************************************\ - * Copyright 2015 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#include "jstatctl.h" -#include "src/common/libutil/oom.h" -#include "src/common/libutil/log.h" -#include "src/common/libutil/xzmalloc.h" -#include "src/common/libutil/iterators.h" -#include "src/common/libutil/lru_cache.h" - -const char *const JSC_JOBID = "jobid"; -const char *const JSC_STATE_PAIR = "state-pair"; -const char *const JSC_STATE_PAIR_OSTATE = "ostate"; -const char *const JSC_STATE_PAIR_NSTATE = "nstate"; -const char *const JSC_RDESC = "rdesc"; -const char *const JSC_RDESC_NNODES = "nnodes"; -const char *const JSC_RDESC_NTASKS = "ntasks"; -const char *const JSC_RDESC_NCORES = "ncores"; -const char *const JSC_RDESC_NGPUS = "ngpus"; -const char *const JSC_RDESC_WALLTIME = "walltime"; -const char *const JSC_RDL = "rdl"; -const char *const JSC_R_LITE = "R_lite"; - -/******************************************************************************* - * * - * Internal User Define Types and Data * - * * - *******************************************************************************/ - -typedef struct { - job_state_t i; - const char *s; -} stab_t; - -typedef struct { - jsc_handler_f cb; - void *arg; -} cb_pair_t; - -typedef struct { - zhash_t *active_jobs; - lru_cache_t *kvs_paths; - flux_msg_handler_t **handlers; - zlist_t *callbacks; - flux_t *h; -} jscctx_t; - -static stab_t job_state_tab[] = { - { J_NULL, "null" }, - { J_RESERVED, "reserved" }, - { J_SUBMITTED, "submitted" }, - { J_PENDING, "pending" }, - { J_SCHEDREQ, "schedreq" }, - { J_SELECTED, "selected" }, - { J_ALLOCATED, "allocated" }, - { J_RUNREQUEST, "runrequest" }, - { J_STARTING, "starting" }, - { J_SYNC, "sync" }, - { J_RUNNING, "running" }, - { J_CANCELLED, "cancelled" }, - { J_COMPLETING, "completing" }, - { J_COMPLETE, "complete" }, - { J_REAPED, "reaped" }, - { J_FAILED, "failed" }, - { J_FOR_RENT, "for_rent" }, - { -1, NULL }, -}; - - -/****************************************************************************** - * * - * Utilities * - * * - ******************************************************************************/ - -const char *jsc_job_num2state (job_state_t i) -{ - stab_t *ss = job_state_tab; - while (ss->s != NULL) { - if (ss->i == i) - return ss->s; - ss++; - } - return NULL; -} - -int jsc_job_state2num (const char *s) -{ - stab_t *ss = job_state_tab; - while (ss->s != NULL) { - if (!strcmp (ss->s, s)) - return ss->i; - ss++; - } - return -1; -} - -static void freectx (void *arg) -{ - jscctx_t *ctx = arg; - zhash_destroy (&(ctx->active_jobs)); - lru_cache_destroy (ctx->kvs_paths); - zlist_destroy (&(ctx->callbacks)); - flux_msg_handler_delvec (ctx->handlers); - free (ctx); -} - -static jscctx_t *getctx (flux_t *h) -{ - jscctx_t *ctx = (jscctx_t *)flux_aux_get (h, "jstatctrl"); - if (!ctx) { - ctx = xzmalloc (sizeof (*ctx)); - if (!(ctx->active_jobs = zhash_new ())) - oom (); - if (!(ctx->kvs_paths = lru_cache_create (256))) - oom (); - lru_cache_set_free_f (ctx->kvs_paths, free); - if (!(ctx->callbacks = zlist_new ())) - oom (); - ctx->h = h; - if (flux_aux_set (h, "jstatctrl", ctx, freectx) < 0) - oom (); - } - return ctx; -} - -static int lwj_kvs_path (flux_t *h, int64_t id, char **pathp) -{ - int rc = -1; - const char *path; - flux_future_t *f; - uint32_t nodeid = FLUX_NODEID_ANY; - - if (!(f = flux_rpc_pack (h, "job.kvspath", nodeid, 0, "{s:[I]}", "ids", id)) - || flux_rpc_get_unpack (f, "{s:[s]}", "paths", &path) < 0) - goto out; - if (!(*pathp = strdup (path))) - goto out; - rc = 0; -out: - flux_future_destroy (f); - return (rc); -} - -static int jscctx_add_jobid_path (jscctx_t *ctx, int64_t id, const char *path) -{ - char *s; - char key [21]; - memset (key, 0, sizeof (key)); - if (sprintf (key, "%"PRId64, id) < 0) - return (-1); - if (!(s = strdup (path))) - return (-1); - if (lru_cache_put (ctx->kvs_paths, key, s) < 0) - free (s); - return (0); -} - -static const char * jscctx_jobid_path (jscctx_t *ctx, int64_t id) -{ - char *path; - char key [21]; - - memset (key, 0, sizeof (key)); - if (sprintf (key, "%"PRId64, id) < 0) - return (NULL); - if ((path = lru_cache_get (ctx->kvs_paths, key))) - return path; - if (lwj_kvs_path (ctx->h, id, &path) < 0) - return (NULL); - if (lru_cache_put (ctx->kvs_paths, key, path) < 0) - return (NULL); - return (path); -} - -static int *active_insert (zhash_t *hash, int64_t jobid) -{ - char key[32]; - int *state; - snprintf (key, sizeof (key), "%"PRId64, jobid); - if (!(state = calloc (1, sizeof (int)))) - return NULL; - if (zhash_insert (hash, key, state) < 0) { - free (state); - return NULL; - } - zhash_freefn (hash, key, (zhash_free_fn *)free); - return state; -} - -static int *active_lookup (zhash_t *hash, int64_t jobid) -{ - char key[32]; - snprintf (key, sizeof (key), "%"PRId64, jobid); - return zhash_lookup (hash, key); -} - -static void active_delete (zhash_t *hash, int64_t jobid) -{ - char key[32]; - snprintf (key, sizeof (key), "%"PRId64, jobid); - return zhash_delete (hash, key); -} - -/****************************************************************************** - * * - * Internal JCB Accessors * - * * - ******************************************************************************/ - -static int jobid_exist (flux_t *h, int64_t jobid) -{ - jscctx_t *ctx = getctx (h); - flux_future_t *f = NULL; - const char *path; - int rc = -1; - - if (!active_lookup (ctx->active_jobs, jobid)) { - if (!(path = jscctx_jobid_path (ctx, jobid))) - goto done; - if (!(f = flux_kvs_lookup (h, NULL, FLUX_KVS_READDIR, path)) - || flux_future_get (f, NULL) < 0) - goto done; - } - rc = 0; -done: - flux_future_destroy (f); - return rc; -} - -static char *lwj_vkey (flux_t *h, int64_t jobid, const char *fmt, va_list ap) -{ - jscctx_t *ctx = getctx (h); - const char *base; - char *name; - char *key; - - if (!(base = jscctx_jobid_path (ctx, jobid))) - return (NULL); - if (vasprintf (&name, fmt, ap) < 0) - return (NULL); - if (asprintf (&key, "%s%s", base, name) < 0) { - free (name); - return (NULL); - } - free (name); - return (key); -} - -static char * __attribute__ ((format (printf, 3, 4))) -lwj_key (flux_t *h, int64_t jobid, const char *fmt, ...) -{ - va_list ap; - char *key; - - va_start (ap, fmt); - key = lwj_vkey (h, jobid, fmt, ap); - va_end (ap); - return key; -} - -static flux_future_t * __attribute__ ((format (printf, 3, 4))) -lookup_job_attribute (flux_t *h, int64_t jobid, const char *fmt, ...) -{ - va_list ap; - flux_future_t *f; - char *key; - - va_start (ap, fmt); - key = lwj_vkey (h, jobid, fmt, ap); - va_end (ap); - if (!key) - return NULL; - f = flux_kvs_lookup (h, NULL, 0, key); - free (key); - return f; -} - -/* Old state is unavailable through the query. - * One should use notification service instead. - */ -static json_t *query_state_pair (flux_t *h, int64_t jobid) -{ - json_t *jcb = NULL; - char *key; - int64_t state; - const char *state_str; - flux_future_t *f = NULL; - - if (!(key = lwj_key (h, jobid, ".state"))) - return NULL; - - if (!(f = flux_kvs_lookup (h, NULL, 0, key)) - || flux_kvs_lookup_get_unpack (f, "s", &state_str) < 0) - goto done; - state = jsc_job_state2num (state_str); - if (!(jcb = json_pack ("{s:{s:I s:I}}", - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, state, - JSC_STATE_PAIR_NSTATE, state))) - goto done; -done: - flux_future_destroy (f); - free (key); - return jcb; -} - -static json_t *query_rdesc (flux_t *h, int64_t jobid) -{ - json_t *jcb = NULL; - int64_t nnodes; - int64_t ntasks; - int64_t ncores; - int64_t ngpus; - int64_t walltime; - - if (jsc_query_rdesc_efficiently (h, jobid, &nnodes, &ntasks, &ncores, - &walltime, &ngpus) < 0) - return NULL; - - if (!(jcb = json_pack ("{s:{s:I s:I s:I s:I s:I}}", - JSC_RDESC, - JSC_RDESC_NNODES, nnodes, - JSC_RDESC_NTASKS, ntasks, - JSC_RDESC_NCORES, ncores, - JSC_RDESC_WALLTIME, walltime, - JSC_RDESC_NGPUS, ngpus))) - return NULL; - return jcb; -} - -int jsc_query_rdesc_efficiently (flux_t *h, int64_t jobid, - int64_t *nnodes, int64_t *ntasks, - int64_t *ncores, int64_t *walltime, - int64_t *ngpus) -{ - flux_future_t *f_nnodes = NULL; - flux_future_t *f_ntasks = NULL; - flux_future_t *f_ncores = NULL; - flux_future_t *f_wall = NULL; - flux_future_t *f_ngpus = NULL; - int rc = -1; - - /* Send requests (in parallel) - */ - if (nnodes && !(f_nnodes = lookup_job_attribute (h, jobid, ".nnodes"))) - goto error; - if (ntasks && !(f_ntasks = lookup_job_attribute (h, jobid, ".ntasks"))) - goto error; - if (ncores && !(f_ncores = lookup_job_attribute (h, jobid, ".ncores"))) - goto error; - if (walltime && !(f_wall = lookup_job_attribute (h, jobid, ".walltime"))) - goto error; - if (ngpus && !(f_ngpus = lookup_job_attribute (h, jobid, ".ngpus"))) - goto error; - - /* Handle responses - */ - if (f_nnodes && flux_kvs_lookup_get_unpack (f_nnodes, "I", nnodes) < 0) - goto error; - if (f_ntasks && flux_kvs_lookup_get_unpack (f_ntasks, "I", ntasks) < 0) - goto error; - if (f_ncores && flux_kvs_lookup_get_unpack (f_ncores, "I", ncores) < 0) - goto error; - if (f_wall && flux_kvs_lookup_get_unpack (f_wall, "I", walltime) < 0) - goto error; - if (f_ngpus && flux_kvs_lookup_get_unpack (f_ngpus, "I", ngpus) < 0) - goto error; - - rc = 0; -error: - flux_future_destroy (f_nnodes); - flux_future_destroy (f_ntasks); - flux_future_destroy (f_ncores); - flux_future_destroy (f_wall); - flux_future_destroy (f_ngpus); - return rc; -} - -static json_t *query_rdl (flux_t *h, int64_t jobid) -{ - flux_future_t *f; - const char *json_str; - json_t *rdl; - json_t *jcb = NULL; - - if (!(f = lookup_job_attribute (h, jobid, ".rdl"))) - return NULL; - if (flux_kvs_lookup_get (f, &json_str) < 0) - goto error; - if (!(rdl = json_loads (json_str, 0, NULL))) - goto error; - if (!(jcb = json_pack ("{s:o}", - JSC_RDL, rdl))) { - json_decref (rdl); - goto error; - } -error: - flux_future_destroy (f); - return jcb; -} - -static json_t *query_r_lite (flux_t *h, int64_t jobid) -{ - flux_future_t *f; - const char *json_str; - json_t *r_lite; - json_t *jcb = NULL; - - if (!(f = lookup_job_attribute (h, jobid, ".R_lite"))) - return NULL; - if (flux_kvs_lookup_get (f, &json_str) < 0) - goto error; - if (!(r_lite = json_loads (json_str, 0, NULL))) - goto error; - if (!(jcb = json_pack ("{s:o}", - JSC_R_LITE, r_lite))) { - json_decref (r_lite); - goto error; - } -error: - flux_future_destroy (f); - return jcb; -} - -static int send_state_event (flux_t *h, job_state_t st, int64_t jobid) -{ - flux_msg_t *msg = NULL; - char *topic = NULL; - int rc = -1; - - if (asprintf (&topic, "jsc.state.%s", jsc_job_num2state (st)) < 0) - goto done; - if ((msg = flux_event_pack (topic, "{ s:I }", "jobid", jobid)) == NULL) - goto done; - if (flux_send (h, msg, 0) < 0) - goto done; - rc = 0; -done: - flux_msg_destroy (msg); - free (topic); - return rc; -} - -static int update_state (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - int state; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - - if (json_unpack (jcb, "{s:{s:i}}", - JSC_STATE_PAIR, - JSC_STATE_PAIR_NSTATE, &state) < 0) - goto done; - if ((state >= J_FOR_RENT) || (state < J_NULL)) - goto done; - if (!(key = lwj_key (h, jobid, ".state"))) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "s", jsc_job_num2state (state)) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL,0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - if (send_state_event (h, state, jobid) < 0) - goto done; - rc = 0; -done: - flux_future_destroy (f); - flux_kvs_txn_destroy (txn); - free (key); - return rc; -} - -static int update_rdesc (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - int64_t nnodes; - int64_t ntasks; - int64_t ncores; - int64_t walltime; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - - if (json_unpack (jcb, "{s:{s:I s:I s:I s:I}}", - JSC_RDESC, - JSC_RDESC_NNODES, &nnodes, - JSC_RDESC_NTASKS, &ntasks, - JSC_RDESC_NCORES, &ncores, - JSC_RDESC_WALLTIME, &walltime) < 0) - goto done; - if ((nnodes < 0) || (ntasks < 0) || (ncores < 0) || (walltime < 0)) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (!(key = lwj_key (h, jobid, ".nnodes"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", nnodes) < 0) - goto done; - free (key); - if (!(key = lwj_key (h, jobid, ".ntasks"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", ntasks) < 0) - goto done; - free (key); - if (!(key = lwj_key (h, jobid, ".walltime"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", walltime) < 0) - goto done; - free (key); - if (!(key = lwj_key (h, jobid, ".ncores"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "I", ncores) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL, 0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - rc = 0; -done: - flux_future_destroy (f); - free (key); - flux_kvs_txn_destroy (txn); - return rc; -} - -static int update_rdl (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - json_t *rdl; - - if (json_unpack (jcb, "{s:o}", JSC_RDL, &rdl) < 0) - goto done; - if (!(key = lwj_key (h, jobid, ".rdl"))) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "O", rdl) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL, 0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - rc = 0; -done: - flux_kvs_txn_destroy (txn); - flux_future_destroy (f); - free (key); - return rc; -} - -static int update_r_lite (flux_t *h, int64_t jobid, json_t *jcb) -{ - int rc = -1; - char *key = NULL; - flux_kvs_txn_t *txn = NULL; - flux_future_t *f = NULL; - json_t *r_lite; - - if (json_unpack (jcb, "{s:o}", JSC_R_LITE, &r_lite) < 0) - goto done; - if (!(txn = flux_kvs_txn_create ())) - goto done; - if (!(key = lwj_key (h, jobid, ".R_lite"))) - goto done; - if (flux_kvs_txn_pack (txn, 0, key, "O", r_lite) < 0) - goto done; - if (!(f = flux_kvs_commit (h, NULL, 0, txn)) - || flux_future_get (f, NULL) < 0) - goto done; - rc = 0; -done: - flux_kvs_txn_destroy (txn); - flux_future_destroy (f); - free (key); - return rc; -} - -static json_t *get_update_jcb (flux_t *h, int64_t jobid, const char *state_str) -{ - jscctx_t *ctx = getctx (h); - int *state; - int nstate = jsc_job_state2num (state_str); - int ostate = J_FOR_RENT; - json_t *o; - - if ((state = active_lookup (ctx->active_jobs, jobid))) { - ostate = *state; - if (nstate == J_COMPLETE || nstate == J_FAILED) - active_delete (ctx->active_jobs, jobid); - else - *state = nstate; - } - if (!(o = json_pack ("{s:I s:{s:i s:i}}", JSC_JOBID, jobid, - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, ostate, - JSC_STATE_PAIR_NSTATE, nstate))) - oom (); - return o; -} - - -/****************************************************************************** - * * - * Internal Asynchronous Notification Mechanisms * - * * - ******************************************************************************/ - -static int invoke_cbs (flux_t *h, int64_t j, json_t *jcb, int errnum) -{ - int rc = 0; - cb_pair_t *c = NULL; - jscctx_t *ctx = getctx (h); - for (c = zlist_first (ctx->callbacks); c; c = zlist_next (ctx->callbacks)) { - char *jcb_str = json_dumps (jcb, 0); - if (!jcb_str) - oom (); - if (c->cb (jcb_str, c->arg, errnum) < 0) - rc = -1; - free (jcb_str); - } - return rc; -} - -static json_t *get_reserve_jcb (flux_t *h, int64_t jobid) -{ - jscctx_t *ctx = getctx (h); - json_t *jcb; - int ostate = J_NULL; - int nstate = J_RESERVED; - int *state; - - if (!(jcb = json_pack ("{s:I s{s:i s:i}}", - JSC_JOBID, jobid, - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, ostate, - JSC_STATE_PAIR_NSTATE, nstate))) - goto error; - - if (!(state = active_insert (ctx->active_jobs, jobid))) - goto error; - *state = nstate; - return jcb; - -error: - json_decref (jcb); - return NULL; -} - -static json_t *get_submit_jcb (flux_t *h, const flux_msg_t *msg, int64_t jobid) -{ - jscctx_t *ctx = getctx (h); - int64_t ntasks; - int64_t nnodes; - int64_t ncores; - int64_t ngpus; - int64_t walltime; - int ostate = J_NULL; - int nstate = J_SUBMITTED; - int *state; - json_t *jcb = NULL; - - if (flux_event_unpack (msg, NULL, "{ s:I s:I s:I s:I s:I }", - "ntasks", &ntasks, - "nnodes", &nnodes, - "ncores", &ncores, - "ngpus", &ngpus, - "walltime", &walltime) < 0) - goto error; - if (!(jcb = json_pack ("{s:I s:{s:i s:i} s:{s:I s:I s:I s:I s:I}}", - JSC_JOBID, jobid, - JSC_STATE_PAIR, - JSC_STATE_PAIR_OSTATE, ostate, - JSC_STATE_PAIR_NSTATE, nstate, - JSC_RDESC, - JSC_RDESC_NNODES, nnodes, - JSC_RDESC_NTASKS, ntasks, - JSC_RDESC_NCORES, ncores, - JSC_RDESC_NGPUS, ngpus, - JSC_RDESC_WALLTIME, walltime))) - goto error; - - if (!(state = active_lookup (ctx->active_jobs, jobid))) { - if (!(state = active_insert (ctx->active_jobs, jobid))) - goto error; - } - *state = nstate; - - return jcb; -error: - json_decref (jcb); - return NULL; -} - -static bool job_is_finished (const char *state) -{ - if (strcmp (state, jsc_job_num2state (J_COMPLETE)) == 0 - || strcmp (state, jsc_job_num2state (J_FAILED)) == 0) - return true; - return false; -} - -static void job_state_cb (flux_t *h, flux_msg_handler_t *mh, - const flux_msg_t *msg, void *arg) -{ - jscctx_t *ctx = getctx (h); - json_t *jcb = NULL; - int64_t jobid; - const char *topic = NULL; - const char *state = NULL; - const char *kvs_path = NULL; - int len = 12; - - if (flux_event_unpack (msg, &topic, "{s:I s?:s}", - "jobid", &jobid, - "kvs_path", &kvs_path) < 0) - goto done; - if (kvs_path && jscctx_add_jobid_path (ctx, jobid, kvs_path) < 0) - goto done; - - if (strncmp (topic, "jsc", 3) == 0) - len = 10; - - state = topic + len; - if (strcmp (state, jsc_job_num2state (J_RESERVED)) == 0) - jcb = get_reserve_jcb (h, jobid); - else if (strcmp (state, jsc_job_num2state (J_SUBMITTED)) == 0) - jcb = get_submit_jcb (h, msg, jobid); - else - jcb = get_update_jcb (h, jobid, state); - - (void)invoke_cbs (h, jobid, jcb, (jcb)? 0 : EINVAL); - - if (job_is_finished (state)) - active_delete (ctx->active_jobs, jobid); -done: - json_decref (jcb); - return; -} - -/****************************************************************************** - * * - * Public Job Status and Control API * - * * - ******************************************************************************/ -static const struct flux_msg_handler_spec htab[] = { - { FLUX_MSGTYPE_EVENT, "wreck.state.*", job_state_cb, 0 }, - { FLUX_MSGTYPE_EVENT, "jsc.state.*", job_state_cb, 0 }, - FLUX_MSGHANDLER_TABLE_END -}; - -int jsc_notify_status (flux_t *h, jsc_handler_f func, void *d) -{ - int rc = -1; - cb_pair_t *c = NULL; - jscctx_t *ctx = NULL; - flux_msg_handler_t **handlers; - - if (!func) - goto done; - if (flux_event_subscribe (h, "wreck.state.") < 0) - goto done; - if (flux_event_subscribe (h, "jsc.state.") < 0) - goto done; - if (flux_msg_handler_addvec (h, htab, NULL, &handlers) < 0) - goto done; - - ctx = getctx (h); - ctx->handlers = handlers; - c = (cb_pair_t *) xzmalloc (sizeof(*c)); - c->cb = func; - c->arg = d; - if (zlist_append (ctx->callbacks, c) < 0) - goto done; - zlist_freefn (ctx->callbacks, c, free, true); - rc = 0; - -done: - return rc; -} - -int jsc_query_jcb (flux_t *h, int64_t jobid, const char *key, char **jcb_str) -{ - json_t *jcb = NULL; - - if (!key) - return -1; - if (jobid_exist (h, jobid) < 0) - return -1; - - if (!strcmp (key, JSC_JOBID)) - jcb = json_pack ("{s:I}", JSC_JOBID, jobid); - else if (!strcmp (key, JSC_STATE_PAIR)) - jcb = query_state_pair (h, jobid); - else if (!strcmp (key, JSC_RDESC)) - jcb = query_rdesc (h, jobid); - else if (!strcmp (key, JSC_RDL)) - jcb = query_rdl (h, jobid); - else if (!strcmp (key, JSC_R_LITE)) - jcb = query_r_lite (h, jobid); - - if (!jcb) - return -1; - char *s = json_dumps (jcb, 0); - if (!s) { - json_decref (jcb); - return -1; - } - *jcb_str = s; - return 0; -} - -int jsc_update_jcb (flux_t *h, int64_t jobid, const char *key, - const char *jcb_str) -{ - int rc = -1; - json_t *jcb; - - if (!jcb_str || !(jcb = json_loads (jcb_str, 0, NULL))) - return -1; - if (jobid_exist (h, jobid) != 0) - goto done; - if (!strcmp (key, JSC_STATE_PAIR)) - rc = update_state (h, jobid, jcb); - else if (!strcmp (key, JSC_RDESC)) - rc = update_rdesc (h, jobid, jcb); - else if (!strcmp (key, JSC_RDL)) - rc = update_rdl (h, jobid, jcb); - else if (!strcmp (key, JSC_R_LITE)) - rc = update_r_lite (h, jobid, jcb); -done: - json_decref (jcb); - return rc; -} - -/* - * vi: ts=4 sw=4 expandtab - */ diff --git a/src/common/libjsc/jstatctl.h b/src/common/libjsc/jstatctl.h deleted file mode 100644 index 6d3e1ab0b558..000000000000 --- a/src/common/libjsc/jstatctl.h +++ /dev/null @@ -1,136 +0,0 @@ -/************************************************************\ - * Copyright 2015 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#ifndef _FLUX_CORE_JSTATCTRL_H -#define _FLUX_CORE_JSTATCTRL_H 1 - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/** - * Define the job states (an abstraction independent of - * underlying task and program execution services (RFC 8) - * and scheduler implementation details (e.g., how the - * attributes of a job are stored in KVS.) For more details, - * please refer to README.md - */ -typedef enum { - J_NULL = 0, /*!< The state has yet to be assigned */ - - /* WRECK job initial condition states: - */ - J_RESERVED = 1, /*!< Reserved by the program execution service */ - J_SUBMITTED = 2, /*!< Submitted to the system */ - - /* Scheduler internal states: - */ - J_PENDING = 11, /*!< Pending */ - J_SCHEDREQ = 12, /*!< Resources requested to be selected */ - J_SELECTED = 13, /*!< Assigned to requested resource in RDL */ - J_ALLOCATED = 14, /*!< Got alloc/contained by the program exec service */ - - /* WRECK job execution states: - */ - J_RUNREQUEST= 21, /*!< Requested to be executed */ - J_STARTING = 22, /*!< Starting */ - J_SYNC = 23, /*!< Tasks stopped in exec waiting for a tool */ - J_RUNNING = 24, /*!< Running */ - J_COMPLETING= 26, /*!< Completing (all tasks exited, epilog running) */ - - /* WRECK job terminal states: - */ - J_CANCELLED = 51, /*!< Cancelled (before execution) */ - J_COMPLETE = 52, /*!< Completed */ - J_FAILED = 53, /*!< Failed (before exec) */ - - /* Scheduler post exec states: - */ - J_REAPED = 101, /*!< Reaped */ - J_FOR_RENT = 102, /*!< Space For Rent */ -} job_state_t; - -typedef int (*jsc_handler_f)(const char *base_jcb, void *arg, int errnum); - -/* TODO: find a better way to manage this hierarchical - * JCB attributes space - */ - -extern const char *const JSC_JOBID; -extern const char *const JSC_STATE_PAIR; -extern const char *const JSC_STATE_PAIR_OSTATE; -extern const char *const JSC_STATE_PAIR_NSTATE; -extern const char *const JSC_RDESC; -extern const char *const JSC_RDESC_NNODES; -extern const char *const JSC_RDESC_NTASKS; -extern const char *const JSC_RDESC_NCORES; -extern const char *const JSC_RDESC_NGPUS; -extern const char *const JSC_RDESC_WALLTIME; -extern const char *const JSC_RDL; -extern const char *const JSC_R_LITE; - -/** - * Register a callback to the asynchronous status change notification service. - * "callback" will be invoked when the state of a job changes. The "jobid" - * and "state-pair" will be passed as "base_jcb" into the callback. - * "d" is arbitrary data that will transparently be passed into "callback." - * However, one should pass its flux_t object as part of this callback data. - * Note that the caller must start its reactor to get an asynchronous status - * change notification via "callback." - * One can register mutliple callbacks by calling this function - * multiple times. The callbacks will be invoked in the order - * they are registered. Returns 0 on success; otherwise -1. - */ -int jsc_notify_status (flux_t *h, jsc_handler_f callback, void *d); - -/** - * Query the "key" attribute of JCB of "jobid." The JCB info on this attribute - * will be passed via "jcb." It is the caller's responsibility to free "jcb." - * Returns 0 on success; otherwise -1. - */ -int jsc_query_jcb (flux_t *h, int64_t jobid, const char *key, char **jcb); - - -/** - * Update the "key" attribute of the JCB of "jobid". The top-level attribute - * of "jcb" should be identical to "key". Return 0 on success; otherwise -1. - * This will not release "jcb," so it is the caller's responsibility to - * free "jcb." - */ -int jsc_update_jcb (flux_t *h, int64_t jobid, const char *key, const char *jcb); - - -/** - * A convenience routine (returning the internal state name correponding to "s.") - */ -const char *jsc_job_num2state (job_state_t s); -int jsc_job_state2num (const char *s); - -/** - * Accessor for nnodes, ntasks, ncores, walltime that circumvents - * JSON encode/decode. N.B. this is a workaround for performance issue - * encountered when scheduling high throughput workloads. - */ -int jsc_query_rdesc_efficiently (flux_t *h, int64_t jobid, - int64_t *nnodes, int64_t *ntasks, - int64_t *ncores, int64_t *walltime, - int64_t *ngpus); - -#ifdef __cplusplus -} -#endif - -#endif /*! _FLUX_CORE_JSTATCTRL_H */ - -/* - * vi: ts=4 sw=4 expandtab - */ diff --git a/src/include/flux/core.h b/src/include/flux/core.h index 539a5130c08a..65a80a5ecfa7 100644 --- a/src/include/flux/core.h +++ b/src/include/flux/core.h @@ -16,7 +16,6 @@ #include "src/common/libflux/flux.h" #include "src/common/libkvs/kvs.h" -#include "src/common/libjsc/jstatctl.h" #include "src/common/libjob/job.h" #endif /* FLUX_CORE_H */ diff --git a/src/modules/Makefile.am b/src/modules/Makefile.am index 57e2457b88f9..defc952933d3 100644 --- a/src/modules/Makefile.am +++ b/src/modules/Makefile.am @@ -4,7 +4,6 @@ SUBDIRS = \ kvs \ kvs-watch \ content-sqlite \ - wreck \ cron \ aggregator \ userdb \ diff --git a/src/modules/wreck/.gitignore b/src/modules/wreck/.gitignore deleted file mode 100644 index ea6dea2a1cb6..000000000000 --- a/src/modules/wreck/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/wrexecd diff --git a/src/modules/wreck/Makefile.am b/src/modules/wreck/Makefile.am deleted file mode 100644 index 6d2f89ae2359..000000000000 --- a/src/modules/wreck/Makefile.am +++ /dev/null @@ -1,97 +0,0 @@ -AM_CFLAGS = \ - $(WARNING_CFLAGS) \ - $(CODE_COVERAGE_CFLAGS) - -AM_LDFLAGS = \ - $(CODE_COVERAGE_LIBS) - -AM_CPPFLAGS = \ - -I$(top_srcdir) \ - -I$(top_srcdir)/src/include \ - -I$(top_builddir)/src/common/libflux \ - -I$(top_srcdir)/src/common/liblsd \ - $(LUA_INCLUDE) \ - $(ZMQ_CFLAGS) - -wreckscriptsdir = $(sysconfdir)/wreck/lua.d - -# -# Comms module -# -fluxmod_LTLIBRARIES = \ - job.la - -fluxlibexec_PROGRAMS = \ - wrexecd - -fluxmod_libadd = $(top_builddir)/src/common/libflux-core.la \ - $(top_builddir)/src/common/libflux-internal.la - -noinst_LTLIBRARIES = \ - librcalc.la -librcalc_la_SOURCES = \ - rcalc.c \ - rcalc.h - -job_la_SOURCES = job.c wreck_job.c wreck_job.h -job_la_LDFLAGS = $(AM_LDFLAGS) $(fluxmod_ldflags) -module -job_la_LIBADD = $(fluxmod_libadd) librcalc.la - -wrexecd_SOURCES = \ - wrexecd.c \ - luastack.c \ - luastack.h - -wrexecd_libs = \ - $(top_builddir)/src/bindings/lua/libfluxlua.la \ - $(top_builddir)/src/common/libflux-internal.la \ - $(top_builddir)/src/common/libpmi/libpmi.la \ - $(top_builddir)/src/common/libflux-core.la \ - $(top_builddir)/src/common/libflux-optparse.la - -wrexecd_LDFLAGS = \ - $(AM_LDFLAGS) -export-dynamic - -wrexecd_LDADD = \ - librcalc.la \ - $(wrexecd_libs) \ - $(ZMQ_LIBS) $(LUA_LIB) $(LIBPTHREAD) $(HWLOC_LIBS) - -TESTS = \ - test_wreck_job.t - -test_ldadd = \ - $(top_builddir)/src/common/libflux-internal.la \ - $(top_builddir)/src/common/libflux-core.la \ - $(top_builddir)/src/common/libtap/libtap.la \ - $(ZMQ_LIBS) $(LIBPTHREAD) - -test_cppflags = \ - $(AM_CPPFLAGS) \ - -I$(top_srcdir)/src/common/libtap - -check_PROGRAMS = $(TESTS) - -test_wreck_job_t_SOURCES = test/wreck_job.c -test_wreck_job_t_CPPFLAGS = $(test_cppflags) -test_wreck_job_t_LDADD = \ - $(top_builddir)/src/modules/wreck/wreck_job.o \ - $(test_ldadd) - -dist_wreckscripts_SCRIPTS = \ - lua.d/01-env.lua \ - lua.d/timeout.lua \ - lua.d/output.lua \ - lua.d/epilog.lua \ - lua.d/input.lua \ - lua.d/mvapich.lua \ - lua.d/pmi-mapping.lua \ - lua.d/intel_mpi.lua \ - lua.d/openmpi.lua \ - lua.d/spectrum.lua \ - lua.d/cuda_devices.lua - -# XXX: Hack below to force rebuild of unbuilt wrexecd dependencies -# -$(wrexecd_libs): - @cd `dirname $@` && $(MAKE) `basename $@` diff --git a/src/modules/wreck/job.c b/src/modules/wreck/job.c deleted file mode 100644 index 328817e269b3..000000000000 --- a/src/modules/wreck/job.c +++ /dev/null @@ -1,939 +0,0 @@ -/************************************************************\ - * Copyright 2014 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include - -#include -#include -#include -#include -#if WITH_TCMALLOC -#if HAVE_GPERFTOOLS_HEAP_PROFILER_H - #include -#elif HAVE_GOOGLE_HEAP_PROFILER_H - #include -#else - #error gperftools headers not configured -#endif -#endif /* WITH_TCMALLOC */ - -#include - -#include - -#include "src/common/libutil/log.h" -#include "src/common/libutil/fdwalk.h" -#include "src/common/libsubprocess/subprocess.h" - -#include "rcalc.h" -#include "wreck_job.h" - -#define MAX_JOB_PATH 1024 - -/* - * lwj directory hierarchy parameters: - * XXX: should be in module context struct perhaps, but - * this all needs to go away soon anyway, so... - * - * directory levels is the number of parent directories - * (e.g. 3 would result in lwj-active.x.y.z.jobid, - * 0 is lwj.jobid) - * - * bits_per_directory is the number of prefix bits to use - * for each parent directory, results in 2^bits entries - * per subdirectory, except for the top-level which will - * grow without bound (well up to 64bit lwj id values). - * - * These values can be set as broker attrs during flux-start, - * e.g. flux start -o,-Swreck.lwj-dir-levels=3 - * -o,-Swreck.lwj-bits-per-dir=8 - */ -static int kvs_dir_levels = 2; -static int kvs_bits_per_dir = 7; - -uint32_t broker_rank; -const char *local_uri = NULL; - -zhash_t *active_jobs = NULL; - -/* - * Return as 64bit integer the portion of integer `n` - * masked from bit position `a` to position `b`, - * then subsequently shifted by `a` bits (to keep - * numbers small). - */ -static inline uint64_t prefix64 (uint64_t n, int a, int b) -{ - uint64_t mask = ((-1ULL >> (64 - b)) & ~((1ULL << a) - 1)); - return ((n & mask) >> a); -} - -/* - * Convert lwj id to kvs path under `lwj-active` using a kind of - * prefix hiearchy of max levels `levels`, using `bits_per_dir` bits - * for each directory. Returns a kvs key path or NULL on failure. - */ -static char * lwj_to_path (uint64_t id, int levels, int bits_per_dir) -{ - char buf [MAX_JOB_PATH] = "lwj"; - int len = 3; - int nleft = sizeof (buf) - len; - int i, n; - - /* Build up kvs directory from lwj. down */ - for (i = levels; i > 0; i--) { - int b = bits_per_dir * i; - uint64_t d = prefix64 (id, b, b + bits_per_dir); - if ((n = snprintf (buf+len, nleft, ".%"PRIu64, d)) < 0 || n > nleft) - return NULL; - len += n; - nleft -= n; - } - n = snprintf (buf+len, sizeof (buf) - len, ".%"PRIu64, id); - if (n < 0 || n > nleft) - return NULL; - return (strdup (buf)); -} - -static char * id_to_path (uint64_t id) -{ - return (lwj_to_path (id, kvs_dir_levels, kvs_bits_per_dir)); -} - -static flux_future_t *next_jobid (flux_t *h) -{ - flux_future_t *f; - - f = flux_rpc_pack (h, "seq.fetch", 0, 0, "{s:s,s:i,s:i,s:b}", - "name", "lwj", - "preincrement", 1, - "postincrement", 0, - "create", true); - return f; -} - -static int next_jobid_get (flux_future_t *f, int64_t *id) -{ - int64_t jobid; - if (flux_rpc_get_unpack (f, "{s:I}", "value", &jobid) < 0) - return -1; - *id = jobid; - return 0; -} - -static char * realtime_string (char *buf, size_t sz) -{ - struct timespec tm; - clock_gettime (CLOCK_REALTIME, &tm); - memset (buf, 0, sz); - snprintf (buf, sz, "%ju.%06ld", (uintmax_t) tm.tv_sec, tm.tv_nsec/1000); - return (buf); -} - -/* Publish wreck.state. event. - * Future is fulfilled once event has received a sequence number. - * This ensures that response to job create request is not sent until this - * happens. See issue #337. - */ -static flux_future_t *send_create_event (flux_t *h, struct wreck_job *job) -{ - char topic[64]; - flux_future_t *f; - int flags = 0; - - if (snprintf (topic, sizeof (topic), "wreck.state.%s", job->state) - >= sizeof (topic)) { - errno = EINVAL; - return NULL; - } - if (!(f = flux_event_publish_pack (h, topic, flags, - "{s:I,s:s,s:i,s:i,s:i,s:i,s:i}", - "jobid", job->id, - "kvs_path", job->kvs_path, - "ntasks", job->ntasks, - "ncores", job->ncores, - "nnodes", job->nnodes, - "ngpus", job->ngpus, - "walltime", job->walltime))) - return NULL; - return f; -} - -static int add_jobinfo_txn (flux_kvs_txn_t *txn, struct wreck_job *job) -{ - char buf [64]; - const char *json_str; - char key[MAX_JOB_PATH]; - size_t keysz = sizeof (key); - flux_msg_t *msg = wreck_job_get_aux (job); - - const char *k; - json_t *v; - json_t *o = NULL; - - if (flux_request_decode (msg, NULL, &json_str) < 0) - goto error; - if (!json_str || !(o = json_loads (json_str, 0, NULL))) - goto inval; - if (snprintf (key, sizeof (key), "%s.state", job->kvs_path) >= sizeof (key)) - goto inval; - if (flux_kvs_txn_pack (txn, 0, key, "s", job->state) < 0) - goto error; - - json_object_foreach (o, k, v) { - int rc; - char *s; - if (snprintf (key, keysz, "%s.%s", job->kvs_path, k) >= keysz) - goto inval; - if (!(s = json_dumps (v, JSON_COMPACT|JSON_ENCODE_ANY))) - goto error; - rc = flux_kvs_txn_put (txn, 0, key, s); - free (s); - if (rc < 0) - goto error; - } - if (snprintf (key, sizeof (key), "%s.create-time", job->kvs_path) - >= sizeof (key)) - goto inval; - if (flux_kvs_txn_pack (txn, 0, key, "s", - realtime_string (buf, sizeof (buf))) < 0) - goto error; - json_decref (o); - return 0; -inval: - errno = EINVAL; -error: - json_decref (o); - return -1; -} - -static bool ping_sched (flux_t *h) -{ - bool retval = false; - flux_future_t *f; - if (!(f = flux_rpc_pack (h, "sched.ping", 0, 0, "{s:i}", "seq", 0))) { - flux_log_error (h, "ping_sched"); - goto out; - } - if (flux_future_get (f, NULL) >= 0) - retval = true; -out: - flux_future_destroy (f); - return (retval); -} - -static bool sched_loaded (flux_t *h) -{ - static bool v = false; - if (!v && ping_sched (h)) - v = true; - return (v); -} - -/* - * Respond to job.submit-nocreate rpc, which allows a job previously - * created by "job.create" to be submitted (The "job.submit" rpc - * creates *and* submits a job). Set the job state to "submitted" - * and issue the wreck.state.submitted event with appropriate payload. - * - * This rpc is used by flux-wreckrun when a scheduler is loaded so that - * "interactive" jobs are subject to normal scheduling. - * (The -I, --immediate flag of flux-wreckrun can be used to disable - * this behavior) - */ -static void job_submit_only (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - int64_t jobid; - struct wreck_job *job = NULL; - flux_future_t *f = NULL;; - - if (!sched_loaded (h)) { - errno = ENOSYS; - goto error; - } - if (flux_request_unpack (msg, NULL, "{s:I}", "jobid", &jobid) < 0) - goto error; - - if (!(job = wreck_job_lookup (jobid, active_jobs))) { - errno = ENOENT; - goto error; - } - wreck_job_set_state (job, "submitted"); - if (!(f = send_create_event (h, job))) - goto error; - if (flux_future_get (f, NULL) < 0) - goto error; - if (flux_respond_pack (h, msg, "{s:I}", "jobid", job->id) < 0) - flux_log_error (h, "flux_respond"); - flux_future_destroy (f); - return; -error: - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "flux_respond"); - flux_future_destroy (f); -} - -/* Handle request to broadcast wreck.state. event. - * This concludes the continuation chain started at job_create_cb(). - * Respond to the original request and destroy 'job'. - */ -static void job_create_event_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - flux_msg_t *msg = wreck_job_get_aux (job); - - if (flux_future_get (f, NULL) < 0) { - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - } - else { - if (flux_respond_pack (h, msg, "{s:I,s:s,s:s}", - "jobid", job->id, - "state", job->state, - "kvs_path", job->kvs_path) < 0) - flux_log_error (h, "flux_respond_pack"); - } - flux_future_destroy (f); -} - - -/* Handle KVS commit response, then send request to broadcast - * wreck.state. event. - * Function is continued in job_create_event_continuation(). - */ -static void job_create_kvs_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - flux_msg_t *msg = wreck_job_get_aux (job); - flux_future_t *f_next = NULL; - - if (flux_future_get (f, NULL) < 0) - goto error; - - /* Preemptively insert this job into the active job hash on this - * node, making it available for use by job_submit_only (). We do - * this *before* we send the event so we avoid racing with the - * event handler that also inserts active jobs. - */ - if (wreck_job_insert (job, active_jobs) < 0) - goto error; - if (!(f_next = send_create_event (h, job))) - goto error; - if (flux_future_then (f_next, -1., job_create_event_continuation, job) < 0) - goto error; - flux_future_destroy (f); - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - flux_future_destroy (f_next); - flux_future_destroy (f); -} - -/* Handle next available jobid response, then issue KVS commit request - * to write job data to KVS. - * Function is continued in job_create_kvs_continuation(). - */ -static void job_create_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - flux_msg_t *msg = wreck_job_get_aux (job); - flux_kvs_txn_t *txn = NULL; - flux_future_t *f_next = NULL; - - if (next_jobid_get (f, &job->id) < 0) - goto error; - if (!(job->kvs_path = id_to_path (job->id))) - goto error; - if (!(txn = flux_kvs_txn_create ())) - goto error; - if (add_jobinfo_txn (txn, job) < 0) - goto error; - if (!(f_next = flux_kvs_commit (h, NULL, 0, txn))) - goto error; - if (flux_future_then (f_next, -1., job_create_kvs_continuation, job) < 0) - goto error; - flux_log (h, LOG_DEBUG, "Setting job %lld to %s", (long long)job->id, - job->state); - flux_kvs_txn_destroy (txn); - flux_future_destroy (f); - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - flux_kvs_txn_destroy (txn); - flux_future_destroy (f_next); - flux_future_destroy (f); - wreck_job_destroy (job); -} - -/* Handle job.create and job.submit requests. - * Create 'job', then send request for next available jobid. - * Function is continued in job_create_continuation(). - */ -static void job_create_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - const char *topic; - flux_msg_t *cpy; - struct wreck_job *job; - flux_future_t *f = NULL; - - if (!(job = wreck_job_create ())) - goto error; - if (flux_request_unpack (msg, &topic, "{s?:i s?:i s?:i s?:i s?:i}", - "ntasks", &job->ntasks, - "nnodes", &job->nnodes, - "ncores", &job->ncores, - "ngpus", &job->ngpus, - "walltime", &job->walltime) < 0) - goto error; - if (!(cpy = flux_msg_copy (msg, true))) - goto error; - wreck_job_set_aux (job, cpy, (flux_free_f)flux_msg_destroy); - if (strcmp (topic, "job.create") == 0) - wreck_job_set_state (job, "reserved"); - else if (strcmp (topic, "job.submit") == 0) { - if (!sched_loaded (h)) { - errno = ENOSYS; - goto error; - } - wreck_job_set_state (job, "submitted"); - } - if (!(f = next_jobid (h))) - goto error; - if (flux_future_then (f, -1., job_create_continuation, job) < 0) - goto error; - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - wreck_job_destroy (job); - flux_future_destroy (f); -} - -static json_t *json_id_to_json_path (flux_t *h, json_t *value) -{ - int64_t id; - char *path = NULL; - json_t *o = NULL; - int errnum = EPROTO; - - if (!json_is_integer (value) || (id = json_integer_value (value)) < 0) - goto out; - if (!(path = id_to_path (id))) { - errnum = errno; - flux_log (h, LOG_ERR, "kvspath_cb: lwj_to_path failed"); - goto out; - } - if (!(o = json_string (path))) { - errnum = errno; - flux_log_error (h, "kvspath_cb: json_string"); - goto out; - } - errnum = 0; -out: - free (path); - errno = errnum; - return (o); -} - -static void job_kvspath_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - int errnum = EPROTO; - size_t index; - json_t *id_list = NULL; - json_t *paths = NULL; - json_t *value; - - if ((flux_request_unpack (msg, NULL, "{s:o}", "ids", &id_list) < 0) - || !json_is_array (id_list)) { - flux_log_error (h, "kvspath_cb failed to unpack message"); - goto out; - } - - paths = json_array (); - json_array_foreach (id_list, index, value) { - json_t *o = json_id_to_json_path (h, value); - if (o == NULL) { - errnum = errno; - goto out; - } - if (json_array_append_new (paths, o) < 0) { - errnum = errno; - flux_log_error (h, "kvspath_cb: json_array_append_new"); - json_decref (o); - goto out; - } - } - if (flux_respond_pack (h, msg, "{s:O}", "paths", paths) < 0) - flux_log_error (h, "kvspath_cb: flux_respond_pack"); - errnum = 0; -out: - if (errnum && flux_respond (h, msg, errnum, NULL) < 0) - flux_log_error (h, "kvspath_cb: flux_respond"); - json_decref (paths); -} - -static int flux_attr_set_int (flux_t *h, const char *attr, int val) -{ - char buf [16]; - int n = snprintf (buf, sizeof (buf), "%d", val); - if (n < 0 || n >= sizeof (buf)) - return (-1); - return flux_attr_set (h, attr, buf); -} - -static int flux_attr_get_int (flux_t *h, const char *attr, int *valp) -{ - long n; - const char *tmp; - char *p; - - if ((tmp = flux_attr_get (h, attr)) == NULL) - return (-1); - n = strtoul (tmp, &p, 10); - if (n == LONG_MAX) - return (-1); - if ((p == tmp) || (*p != '\0')) { - errno = EINVAL; - return (-1); - } - *valp = (int) n; - return (0); -} - -static void completion_cb (flux_subprocess_t *p) -{ - flux_t *h; - struct wreck_job *job = NULL; - int tmp; - - h = flux_subprocess_aux_get (p, "handle"); - job = flux_subprocess_aux_get (p, "job"); - - assert (h && job); - - /* skip output text if exit code == 0 */ - if (!(tmp = flux_subprocess_exit_code (p))) - goto cleanup; - - if (tmp > 0) { - flux_log_error (h, "job%ju: wrexecd: Exit %d", - (uintmax_t) job->id, tmp); - } - else if (tmp < 0) { - if ((tmp = flux_subprocess_signaled (p)) < 0) - flux_log_error (h, "job%ju: unknown exit status", (uintmax_t) job->id); - else - flux_log_error (h, "job%ju: wrexecd: %s", - (uintmax_t) job->id, strsignal (tmp)); - - } - -cleanup: - wreck_job_destroy (job); - flux_subprocess_destroy (p); -} - -static void state_change_cb (flux_subprocess_t *p, flux_subprocess_state_t state) -{ - flux_t *h; - struct wreck_job *job; - - h = flux_subprocess_aux_get (p, "handle"); - job = flux_subprocess_aux_get (p, "job"); - - assert (h && job); - - // XXX: Update job state to failed - if (state == FLUX_SUBPROCESS_EXEC_FAILED) { - flux_log_error (h, "spawn: job%ju: wrexecd exec failure", (uintmax_t) job->id); - flux_subprocess_destroy (p); - } - else if (state == FLUX_SUBPROCESS_FAILED) { - flux_log_error (h, "spawn: job%ju: wrexecd failure", (uintmax_t) job->id); - flux_subprocess_destroy (p); - } -} - -static void io_cb (flux_subprocess_t *p, const char *stream) -{ - int lenp = 0; - const char *ptr; - - if ((ptr = flux_subprocess_read_line (p, stream, &lenp)) - && lenp > 0) { - flux_t *h; - struct wreck_job *job; - int level = LOG_INFO; - - h = flux_subprocess_aux_get (p, "handle"); - job = flux_subprocess_aux_get (p, "job"); - - assert (h && job); - - if (!strcasecmp (stream, "STDERR")) - level = LOG_ERR; - - flux_log (h, level, - "job%ju: wrexecd says: %s", - (uintmax_t) job->id, ptr); - } -} - -static flux_cmd_t *wrexecd_cmd_create (flux_t *h, struct wreck_job *job) -{ - flux_cmd_t *cmd = NULL; - const char *wrexecd_path; - char *cwd = NULL; - char buf [4096]; - int n; - - if (!(cmd = flux_cmd_create (0, NULL, NULL))) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_create"); - goto error; - } - if (!(wrexecd_path = flux_attr_get (h, "wrexec.wrexecd_path"))) { - flux_log_error (h, "wrexecd_cmd_create: flux_attr_get"); - goto error; - } - if (flux_cmd_argv_append (cmd, "%s", wrexecd_path) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_argv_append"); - goto error; - } - n = snprintf (buf, sizeof(buf), "--lwj-id=%ju", (uintmax_t) job->id); - if ((n >= sizeof (buf)) || (n < 0)) { - flux_log_error (h, "failed to append id to cmdline for job%ju\n", - (uintmax_t) job->id); - goto error; - } - if (flux_cmd_argv_append (cmd, "%s", buf) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_argv_append"); - goto error; - } - n = snprintf (buf, sizeof (buf), "--kvs-path=%s", job->kvs_path); - if ((n >= sizeof (buf)) || (n < 0)) { - flux_log_error (h, "failed to append kvspath to cmdline for job%ju\n", - (uintmax_t) job->id); - goto error; - } - if (flux_cmd_argv_append (cmd, "%s", buf) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_argv_append"); - goto error; - } - /* flux_rexec() requires cwd to be set */ - if (!(cwd = get_current_dir_name ())) { - flux_log_error (h, "wrexecd_cmd_create: get_current_dir_name"); - goto error; - } - if (flux_cmd_setcwd (cmd, cwd) < 0) { - flux_log_error (h, "wrexecd_cmd_create: flux_cmd_setcwd"); - goto error; - } - - free (cwd); - return (cmd); -error: - free (cwd); - flux_cmd_destroy (cmd); - return (NULL); -} - -static int spawn_exec_handler (flux_t *h, struct wreck_job *job) -{ - flux_cmd_t *cmd = NULL; - flux_subprocess_t *p = NULL; - flux_subprocess_ops_t ops = { - .on_completion = completion_cb, - .on_state_change = state_change_cb, - .on_channel_out = NULL, - .on_stdout = io_cb, - .on_stderr = io_cb - }; - - if (!(cmd = wrexecd_cmd_create (h, job))) { - flux_log_error (h, "wrexecd_cmd_create"); - goto error; - } - - if (!(p = flux_rexec (h, FLUX_NODEID_ANY, 0, cmd, &ops))) { - flux_log_error (h, "flux_rexec"); - goto error; - } - - if (flux_subprocess_aux_set (p, "handle", h, NULL) < 0) { - flux_log_error (h, "flux_subprocess_aux_set"); - goto error; - } - - if (flux_subprocess_aux_set (p, "job", job, NULL) < 0) { - flux_log_error (h, "flux_subprocess_aux_set"); - goto error; - } - - /* Take a reference on this job since it is now embedded in - * a flux rexec. - */ - flux_cmd_destroy (cmd); - wreck_job_incref (job); - return (0); - -error: - flux_cmd_destroy (cmd); - flux_subprocess_destroy (p); - return (-1); -} - -static bool Rlite_targets_this_node (flux_t *h, const char *key, - const char *R_lite) -{ - rcalc_t *r = NULL; - bool result; - - if (!(r = rcalc_create (R_lite))) { - if (broker_rank == 0) - flux_log (h, LOG_ERR, "Unable to parse %s", key); - return false; - } - result = rcalc_has_rank (r, broker_rank); - rcalc_destroy (r); - return result; -} - -/* Handle response to lookup of R_lite. If this node is targetted, - * spawn wrexecd. If R_lite doesn't exist, fallback to old method - * of looking up rank.N, with one more continuation. - */ -static void runevent_continuation (flux_future_t *f, void *arg) -{ - struct wreck_job *job = arg; - flux_t *h = flux_future_get_flux (f); - const char *key = flux_kvs_lookup_get_key (f); - const char *R_lite; - - if (flux_kvs_lookup_get (f, &R_lite) < 0) { - if (broker_rank == 0) - flux_log (h, LOG_INFO, "No %s: %s", key, flux_strerror (errno)); - goto done; - } - if (!Rlite_targets_this_node (h, key, R_lite)) - goto done; - - if (spawn_exec_handler (h, job) < 0) - goto done; -done: - flux_future_destroy (f); -} - -static int64_t id_from_tag (const char *tag) -{ - unsigned long l; - char *p; - errno = 0; - l = strtoul (tag, &p, 10); - if (*p != '\0') - return (-1); - if (l == 0 && errno == EINVAL) - return (-1); - else if (l == ULONG_MAX && errno == ERANGE) - return (-1); - return l; -} - -/* Handle wrexec.run. event. - * Determine if assigned resources are on this broker rank, then spawn - * wrexecd if so. This function sends request to read R_lite, - * then continues in runevent_continuation(). - */ -static void runevent_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, - void *arg) -{ - int64_t id; - const char *topic; - struct wreck_job *job; - flux_future_t *f = NULL; - char k[MAX_JOB_PATH]; - - if (flux_event_decode (msg, &topic, NULL) < 0) - goto error; - id = id_from_tag (topic+11); - if (!(job = wreck_job_lookup (id, active_jobs))) { - errno = ENOENT; - goto error; - } - if (!job->kvs_path) - goto error; - if (snprintf (k, sizeof (k), "%s.R_lite", job->kvs_path) >= sizeof (k)) { - errno = EINVAL; - goto error; - } - if (!(f = flux_kvs_lookup (h, NULL, 0, k))) - goto error; - if (flux_future_then (f, -1., runevent_continuation, job) < 0) - goto error; - /* N.B. 'f' and 'job' are destroyed by runevent_continuation() */ - return; -error: - flux_log_error (h, "%s", __FUNCTION__); - flux_future_destroy (f); -} - -/* Track job state transition in active_jobs hash - * Currently only id, kvs_path, and state are tracked. - */ -static void wreck_state_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - int64_t id; - const char *topic; - const char *kvs_path = NULL; - struct wreck_job *job; - - if (flux_event_unpack (msg, &topic, "{s:I s?s}", - "jobid", &id, - "kvs_path", &kvs_path) < 0) - goto error; - topic += 12; // state comes after "wreck.state." (12 chars) - if (strlen (topic) == 0 || strlen (topic) >= sizeof (job->state)) { - errno = EPROTO; - goto error; - } - if (!(job = wreck_job_lookup (id, active_jobs))) { - if (!(job = wreck_job_create ())) - goto error; - job->id = id; - if (kvs_path && !(job->kvs_path = strdup (kvs_path))) - goto error_destroy; - if (wreck_job_insert (job, active_jobs) < 0) - goto error_destroy; - } - wreck_job_set_state (job, topic); - if ( !strcmp (job->state, "complete") - || !strcmp (job->state, "failed") - || !strcmp (job->state, "cancelled")) - wreck_job_delete (id, active_jobs); - return; -error_destroy: - wreck_job_destroy (job); -error: - flux_log_error (h, "%s", __FUNCTION__); -} - -static void job_list_cb (flux_t *h, flux_msg_handler_t *w, - const flux_msg_t *msg, void *arg) -{ - char *json_str = NULL; - int max = 0; - const char *include = NULL; - const char *exclude = NULL; - - if (flux_request_unpack (msg, NULL, "{s?:i s?:s s?:s}", - "max", &max, - "include", &include, - "exclude", &exclude) < 0) - goto error; - if (!(json_str = wreck_job_list (active_jobs, max, include, exclude))) - goto error; - if (flux_respond (h, msg, 0, json_str) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); - free (json_str); - return; -error: - if (flux_respond (h, msg, errno, NULL) < 0) - flux_log_error (h, "%s: flux_respond", __FUNCTION__); -} - -static const struct flux_msg_handler_spec mtab[] = { - { FLUX_MSGTYPE_REQUEST, "job.create", job_create_cb, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.submit", job_create_cb, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.submit-nocreate", job_submit_only, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.kvspath", job_kvspath_cb, 0 }, - { FLUX_MSGTYPE_REQUEST, "job.list", job_list_cb, 0 }, - { FLUX_MSGTYPE_EVENT, "wrexec.run.*", runevent_cb, 0 }, - { FLUX_MSGTYPE_EVENT, "wreck.state.*", wreck_state_cb, 0 }, - FLUX_MSGHANDLER_TABLE_END -}; - -int mod_main (flux_t *h, int argc, char **argv) -{ - flux_msg_handler_t **handlers = NULL; - int rc = -1; - - if (!(active_jobs = zhash_new ())) { - flux_log_error (h, "zhash_new"); - return (-1); - } - if (flux_msg_handler_addvec (h, mtab, NULL, &handlers) < 0) { - flux_log_error (h, "flux_msg_handler_addvec"); - goto done; - } - if ((flux_event_subscribe (h, "wrexec.run.") < 0)) { - flux_log_error (h, "flux_event_subscribe"); - goto done; - } - if ((flux_event_subscribe (h, "wreck.state.") < 0)) { - flux_log_error (h, "flux_event_subscribe"); - goto done; - } - - if ((flux_attr_get_int (h, "wreck.lwj-dir-levels", &kvs_dir_levels) < 0) - && (flux_attr_set_int (h, "wreck.lwj-dir-levels", kvs_dir_levels) < 0)) { - flux_log_error (h, "failed to get or set lwj-dir-levels"); - goto done; - } - if ((flux_attr_get_int (h, "wreck.lwj-bits-per-dir", - &kvs_bits_per_dir) < 0) - && (flux_attr_set_int (h, "wreck.lwj-bits-per-dir", - kvs_bits_per_dir) < 0)) { - flux_log_error (h, "failed to get or set lwj-bits-per-dir"); - goto done; - } - - if (flux_get_rank (h, &broker_rank) < 0) { - flux_log_error (h, "flux_get_rank"); - goto done; - } - - if (!(local_uri = flux_attr_get (h, "local-uri"))) { - flux_log_error (h, "flux_attr_get (\"local-uri\")"); - goto done; - } - - if (flux_reactor_run (flux_get_reactor (h), 0) < 0) { - flux_log_error (h, "flux_reactor_run"); - goto done; - } - rc = 0; -done: - flux_msg_handler_delvec (handlers); - zhash_destroy (&active_jobs); - return rc; -} - -MOD_NAME ("job"); - -/* - * vi: ts=4 sw=4 expandtab - */ diff --git a/src/modules/wreck/lua.d/01-env.lua b/src/modules/wreck/lua.d/01-env.lua deleted file mode 100644 index 2de1374a20bf..000000000000 --- a/src/modules/wreck/lua.d/01-env.lua +++ /dev/null @@ -1,57 +0,0 @@ - --- --- Set WRECK environment from environment table [environ] --- -local function rexec_set_env (environ) - if not environ then return end - - local env = wreck.environ - for k,v in pairs (environ) do - env[k] = v - end -end --- --- Set common environment and working directory for all tasks --- -function rexecd_init () - local posix = require 'flux.posix' - - -- Initialize environment with top level lwj.environ - local k = wreck.flux:kvsdir("lwj") - rexec_set_env (k.environ) - - -- Get kvsdir handle to this LWJ's entry in KVS and override - -- with lwj..environ - -- - local lwj = wreck.kvsdir - rexec_set_env (lwj.environ) - - -- Check for current working directory as lwj..cwd - local cwd = lwj.cwd - if cwd then - posix.chdir (cwd) - end -end - --- --- Per-task version of above: --- If lwj..environ or cwd exist then these override settings --- from lwj.{environ,cwd}: --- -function rexecd_task_init () - local posix = require 'flux.posix' - local taskid = wreck.taskid; - - local task = wreck.by_task - if not task then return end - - -- Set any task-specific environ from lwj...environ - rexec_set_env (task.environ) - - -- Each task can have a different cwd: - local cwd = task.cwd - if cwd then - posix.chdir (cwd) - end -end --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/cuda_devices.lua b/src/modules/wreck/lua.d/cuda_devices.lua deleted file mode 100644 index ea5da88837fd..000000000000 --- a/src/modules/wreck/lua.d/cuda_devices.lua +++ /dev/null @@ -1,75 +0,0 @@ -local gpubind = wreck:getopt ("gpubind") -if gpubind == "no" or gpubind == "off" then - return -end - --- Set CUDA_VISIBLE_DEVICES for all tasks on any rank with one or --- more "gpu" resources - -local gpuinfo = {} -function gpuinfo_create (wreck, gpus) - local g = {} - -- Use affinity.cpuset as a convenience to parse the GPU list, which - -- is in nodeset form (e.g. "0-1" or "0,2-5", etc.) - -- - local gset, err = require 'flux.affinity'.cpuset.new (gpus) - if not gset then - wreck:log_error ("Unable to parse GPU list [%s]: %s", gpus, err) - return nil - end - local g = { - gpuids = gset:expand (), - ngpus = gset:count (), - ntasks = wreck.tasks_per_node [wreck.nodeid] - } - - -- If per-task binding is requested, ensure ngpus is evenly divisible - -- into ntasks: - if gpubind == "per-task" and g.ngpus % g.ntasks == 0 then - g.ngpus_per_task = g.ngpus/g.ntasks - end - return g -end - -function rexecd_init () - -- NB: Lua arrays are indexed starting at 1, so this rank's index - -- into R_lite rank array is nodeid + 1: - -- - local index = wreck.nodeid + 1 - - -- Grab local resources structure from kvs for this nodeid: - -- - local Rlocal = wreck.kvsdir.R_lite[index].children - - -- If a gpu resource list is set for this rank, then expand it and - -- set CUDA_VISIBLE_DEVICES to the result: - -- - local gpus = Rlocal.gpu - if not gpus then return end - - gpuinfo = gpuinfo_create (wreck, gpus) - -- If ngpus_per_task is not set, then set CUDA_VISIBLE_DEVICES the same - -- for all tasks: - if not gpuinfo.ngpus_per_task then - local ids = table.concat (gpuinfo.gpuids, ",") - wreck.environ ["CUDA_VISIBLE_DEVICES"] = ids - end - -- Always set CUDA_DEVICE_ORDER=PCI_BUS_ID to ensure CUDA ids match - -- IDs known to flux scheduler. - wreck.environ ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" -end - -function rexecd_task_init () - -- If ngpus_per_task is set, then select that many GPUs from the gpuids - -- list assigned to this rank for the current task: - if not gpuinfo.ngpus_per_task then return end - - local basis = gpuinfo.ngpus_per_task * wreck.taskid - local t = {} - for i = 1,gpuinfo.ngpus_per_task do - table.insert (t, gpuinfo.gpuids [basis + i]) - end - wreck.environ ["CUDA_VISIBLE_DEVICES"] = table.concat (t, ",") -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/epilog.lua b/src/modules/wreck/lua.d/epilog.lua deleted file mode 100644 index c7a32b41b8a8..000000000000 --- a/src/modules/wreck/lua.d/epilog.lua +++ /dev/null @@ -1,21 +0,0 @@ -local posix = require 'flux.posix' - --- execute a path from kvs `key` -local function run_kvs (key) - local epilog = wreck.kvsdir [key] or wreck.flux:kvs_get ("lwj."..key) - if not epilog then return end - return os.execute (epilog) -end - -function rexecd_complete () - local rc, err = run_kvs ("epilog.pre") - if not rc then wreck:log_msg ("error: epilog: %s", err) end -end - --- rexecd_exit callback happens after the job is in the complete state -function rexecd_exit () - local rc, err = run_kvs ("epilog.post") - if not rc then wreck:log_msg ("error: epilog.post: %s", err) end -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/input.lua b/src/modules/wreck/lua.d/input.lua deleted file mode 100644 index 83efcc38ef53..000000000000 --- a/src/modules/wreck/lua.d/input.lua +++ /dev/null @@ -1,202 +0,0 @@ --- --- Process any stdin configuration for a job, opening files --- if requested, and setting up task stdin links as --- directed in lwj..input.config --- -local posix = require 'flux.posix' - --- temporary "set" class until we have nodeset lua bindings -local taskset = {} -function taskset:__index (key) - if tonumber (key) then - return self.t [tostring(key)] - end - return rawget (taskset,key) -end -function taskset.new (arg) - local hostlist = require 'flux.hostlist' - local t = {} - local hl = arg and hostlist.new ("["..arg.."]") or hostlist.new () - for i in hl:next () do - t[tostring(i)] = true - end - return setmetatable ({ t = t }, taskset) -end -function taskset:len () - local count = 0 - for i,v in pairs (self.t) do - count = count + 1 - end - return count -end - -function taskset:set (i) - local k = tostring (i) - local prev = self.t[k] - self.t[k] = true - return prev or false -end - -function taskset:clear (i) - local k = tostring (i) - local prev = self.t[k] - self.t[k] = nil - return prev or false -end - -function taskset:iterator () - return pairs (self.t) -end - ----- - -local function kvs_input_config (wreck) - local o = wreck.kvsdir ["input.config"] - if not o then - o = wreck.flux:kvsdir ("lwj.input.config") - end - return o -end - -local inputconf = {} -inputconf.__index = inputconf -function inputconf.create (wreck) - local c = { - conf = kvs_input_config (wreck), - wreck = wreck, - maxid = wreck.kvsdir.ntasks - 1, - basedir = tostring (wreck.kvsdir), - inputsdir = tostring (wreck.kvsdir) .. ".input.files", - inputs = {}, - } - -- Add default input config - if not c.conf then - c.conf = { { src = "stdin", dst = "*" } } - end - - c.tasksleft, err = taskset.new ('0-'..c.maxid) - - -- Add stdin entry to inputs - c.inputs[1] = { filename = "stdin", - kzpath = c.inputsdir .. ".stdin" } - return setmetatable (c, inputconf) -end - -function inputconf:set (taskid) - if self.tasksleft[taskid] then - self.tasksleft:clear(taskid) - return false - end - return true -end - -function inputconf:add_source (path) - local i = { filename = path } - local f = self.wreck.flux - - -- First open file for input: - local fp, err = io.open (path, "r") - if not fp then return nil, err end - i.fd = posix.fileno (fp) - - -- now open kz stream for writing: - local basedir = self.inputsdir .. "." .. #self.inputs - f:kvs_put (basedir..".filename", i.filename) - i.kzpath = basedir .. ".kz" - local kz, err = f:kz_open (i.kzpath, "w") - if not kz then - io.close (fp) - return nil, err - end - i.kz = kz - table.insert (self.inputs, i) - return i -end - -function inputconf:input (path) - for _,t in pairs (self.inputs) do - if t.filename == path then - return t - end - end - return self:add_source (path) -end - -function inputconf:render (template, taskid) - local lustache = require 'flux.lustache' - local view = { - taskid = taskid - } - setmetatable (view, { __index = self.wreck }) - return lustache:render (template, view) -end - - -function inputconf:task_input (template, i) - return self:input (self:render (template, i)) -end - -function inputconf:dst_to_list (dst) - if dst == "" or dst == "*" or dst == "all" then - return self.tasksleft - end - return taskset.new (dst) -end - -function inputconf:task_stdin (taskid) - return self.basedir .. "." .. taskid .. ".stdin" -end - --- Link stdin for task id list ids to "path" -function inputconf:link (ids, path) - local f = self.wreck.flux - local taskids = self:dst_to_list (ids) - if taskids:len() == 0 then return true end - for i in taskids:iterator () do - if not self:set (i) then - local input, err = self:task_input (path, i) - if not input then return nil, err end - f:kvs_symlink (self:task_stdin (i), input.kzpath) - end - end - return true -end - -function inputconf:process_config () - for _, input in pairs (self.conf) do - local rc, err = self:link (input.dst, input.src) - if not rc then return nil, err end - end - return true -end -function input_start (f, input) - f:iowatcher { - fd = input.fd, - handler = function (iow, r) - if r.data then input.kz:write (r.data) end - if r.eof then input.kz:close () end - end - } -end - -function inputconf:start () - local f = self.wreck.flux - for path, input in pairs (self.inputs) do - if input.kz and input.fd then - input_start (f, input) - end - end -end -function rexecd_init () - if wreck.nodeid ~= 0 then return end - local f = wreck.flux - local dir = tostring (wreck.kvsdir) - - local cfg = inputconf.create (wreck) - local rc, err = cfg:process_config () - if not rc then - wreck:die ("Error: input: %s", err) - return - end - cfg:start () -end diff --git a/src/modules/wreck/lua.d/intel_mpi.lua b/src/modules/wreck/lua.d/intel_mpi.lua deleted file mode 100644 index c55bc8097f54..000000000000 --- a/src/modules/wreck/lua.d/intel_mpi.lua +++ /dev/null @@ -1,24 +0,0 @@ --- Set environment specific to Intel MPI --- --- Intel PMI is an MPICH derivative that bootstraps with the PMI-1 wire --- protocol, or if I_MPI_PMI_LIBRARY is set, a PMI library. --- --- If the library is set, override it so it points to Flux's. --- If the library is unset, do nothing. --- --- (N.B. We could just unconditionally unset it, but that would prevent the --- user from setting it in order to enable client side PMI tracing in the --- Flux PMI library, enabled by setting FLUX_PMI_DEBUG=1) - - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local libpmi = f:getattr ('conf.pmi_library_path') - - if env['I_MPI_PMI_LIBRARY'] ~= nil then - env['I_MPI_PMI_LIBRARY'] = libpmi - end -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/mvapich.lua b/src/modules/wreck/lua.d/mvapich.lua deleted file mode 100644 index 192c9021b795..000000000000 --- a/src/modules/wreck/lua.d/mvapich.lua +++ /dev/null @@ -1,31 +0,0 @@ --- Set environment specific to mvapich - - -local dirname = require 'flux.posix'.dirname - --- XXX: MVAPICH2 at least requires MPIRUN_RSH_LAUNCH to be set --- in the environment or PMI doesn't work (for unknown reason) --- - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local libpmi = f:getattr ('conf.pmi_library_path') - local ldpath = dirname (libpmi) - - if (env['LD_LIBRARY_PATH'] ~= nil) then - ldpath = ldpath..':'..env['LD_LIBRARY_PATH'] - end - - env['LD_LIBRARY_PATH'] = ldpath - env['MPIRUN_NTASKS'] = wreck.kvsdir.ntasks - env['MPIRUN_RSH_LAUNCH'] = 1 -end - - -function rexecd_task_init () - local env = wreck.environ - env['MPIRUN_RANK'] = wreck.globalid -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/openmpi.lua b/src/modules/wreck/lua.d/openmpi.lua deleted file mode 100644 index 7a5a4c411063..000000000000 --- a/src/modules/wreck/lua.d/openmpi.lua +++ /dev/null @@ -1,16 +0,0 @@ --- Set environment specific to openmpi --- - -local dirname = require 'flux.posix'.dirname - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local rundir = f:getattr ('broker.rundir') - - -- Avoid shared memory segment name collisions - -- when flux instance runs >1 broker per node. - env['OMPI_MCA_orte_tmpdir_base'] = rundir -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/output.lua b/src/modules/wreck/lua.d/output.lua deleted file mode 100644 index 25bb4e71943d..000000000000 --- a/src/modules/wreck/lua.d/output.lua +++ /dev/null @@ -1,126 +0,0 @@ --- --- Optionally write job output to file(s) if so configured in either --- lwj.output or lwj..output --- --- Output filenames are rendered mustache templates --- --- -local ioplex = require 'wreck.io' -local flux = require 'flux' - -local function kvs_output_config (wreck) - local o = wreck.kvsdir ["output"] - if not o then - o = wreck.flux:kvsdir ("lwj.output") - end - return o -end - -local function render (template, wreck, taskid) - local lustache = require 'flux.lustache' - local view = { - cmd = wreck.argv[0]:match("([^/]+)$"), - taskid = taskid - } - setmetatable (view, { __index = wreck }) - return lustache:render (template, view) -end - -local function openstream (wreck, taskio, taskid, stream, template) - local path = render (template, wreck, taskid) - if not path then - wreck:die ("output: Failed to render template '"..template.."'") - return - end - -- Make any kvs:// output relative to the current kvsdir - local key = path:match ("kvs://(.*)$") - if key then - path = "kvs://"..tostring (wreck.kvsdir) .."."..key - end - taskio:redirect (taskid, stream, path) - return path -end - -local function log (fmt, ...) - wreck:log_msg (fmt, ...) -end - -local function log_err (fmt, ...) - wreck:log_error (fmt, ...) -end - --- Read `ioservice` entry for this job, and map any stream with --- rank == FLUX_NODEID_RANK to this rank. --- -local function fetch_ioservices (wreck) - local ioservice,err = wreck.kvsdir.ioservice - if not ioservice then return nil end - local rank = wreck.flux.rank - - -- Only streams with rank == -1 are handled by - -- this plugin. Delete other entries and remap FLUX_NODEID_ANY - -- to this rank: - for s,v in pairs (ioservice) do - if v and v.rank == -1 then - ioservice[s].rank = rank - else - ioservice[s] = nil - end - end - return ioservice -end - -function rexecd_init () - if wreck.nodeid ~= 0 then return end - - local output = kvs_output_config (wreck) - if not output or not output.files then return end - - local template = output.files.stdout - local stderr_template = output.files.stderr - if not template and not stderr_template then return end - - local ntasks = wreck.kvsdir.ntasks - local ioservices = fetch_ioservices (wreck) - - taskio, err = ioplex.create { - flux = wreck.flux, - jobid = wreck.id, - labelio = output.labelio and output.labelio ~= false, - log_err = log_err, - nokz = wreck.kvsdir.options.nokz, - ioservices = ioservices - } - if not taskio then - wreck:log_error ("Error: %s", err) - return - end - - ioplex:enable_debug (log) - - for i = 0, ntasks - 1 do - if template then - openstream (wreck, taskio, i, "stdout", template) - end - if stderr_template then - openstream (wreck, taskio, i, "stderr", stderr_template) - elseif template then - taskio:dup (i, "stderr", "stdout") - end - end - taskio:start () -end - -function rexecd_exit () - if wreck.nodeid ~= 0 or not taskio then return end - while not taskio:complete() do - local rc, err = wreck.flux:reactor ("once") - if not rc then - log_err ("rexecd_exit: reactor once failed: %s", err or "No error") - return - end - end - wreck:log_msg ("File io complete") -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/pmi-mapping.lua b/src/modules/wreck/lua.d/pmi-mapping.lua deleted file mode 100644 index acf3de6ad738..000000000000 --- a/src/modules/wreck/lua.d/pmi-mapping.lua +++ /dev/null @@ -1,38 +0,0 @@ --- Set pmi.PMI_process_mapping - - --- compute blocks list as a Lua table from tasks_per_node and nnodes: -local function compute_blocks (cpn, nnodes) - local blocks = {} - local last = nil - for i = 0, nnodes - 1 do - local count = cpn [i] - if last and cpn[i] == last.tasks then - last.count = last.count + 1 - else - last = { start = i, count = 1, tasks = cpn [i] } - table.insert (blocks, last) - end - end - return blocks -end - --- return 'standard' PMI_process_mapping vector as a string -local function blocks_to_pmi_mapping (blocks) - if not blocks then return " " end - local s = "(vector" - for _,b in pairs (blocks) do - s = s .. string.format (",(%d,%d,%d)", b.start, b.count, b.tasks) - end - s = s .. ")" - return (s) -end - -function rexecd_init () - if (wreck.nodeid ~= 0) then return end - - local blocks = compute_blocks (wreck.tasks_per_node, wreck.nnodes) - local mapping = blocks_to_pmi_mapping (blocks) - wreck.kvsdir ["pmi.PMI_process_mapping"] = mapping -end - diff --git a/src/modules/wreck/lua.d/spectrum.lua b/src/modules/wreck/lua.d/spectrum.lua deleted file mode 100644 index 6985a4a1e984..000000000000 --- a/src/modules/wreck/lua.d/spectrum.lua +++ /dev/null @@ -1,49 +0,0 @@ --- Set environment specific to spectrum_mpi (derived from openmpi) --- - -if wreck:getopt ("mpi") ~= "spectrum" then return end - -local posix = require 'posix' - -function prepend_path (env_var, path) - local env = wreck.environ - if env[env_var] == nil then - suffix = '' - else - suffix = ':'..env[env_var] - end - env[env_var] = path..suffix -end - -function rexecd_init () - local env = wreck.environ - local f = wreck.flux - local rundir = f:getattr ('broker.rundir') - - -- Avoid shared memory segment name collisions - -- when flux instance runs >1 broker per node. - env['OMPI_MCA_orte_tmpdir_base'] = rundir - - -- Assumes the installation paths of Spectrum MPI on LLNL's Sierra - env['OMPI_MCA_osc'] = "pt2pt" - env['OMPI_MCA_pml'] = "yalla" - env['OMPI_MCA_btl'] = "self" - env['MPI_ROOT'] = "/opt/ibm/spectrum_mpi" - env['OPAL_LIBDIR'] = "/opt/ibm/spectrum_mpi/lib" - env['OMPI_MCA_coll_hcoll_enable'] = '0' - - env['PMIX_SERVER_URI'] = nil - env['PMIX_SERVER_URI2'] = nil - - -- Help find libcollectives.so - prepend_path ('LD_LIBRARY_PATH', '/opt/ibm/spectrum_mpi/lib/pami_port') - prepend_path ('LD_PRELOAD', '/opt/ibm/spectrum_mpi/lib/libpami_cudahook.so') -end - -function rexecd_task_init () - -- Approximately `ulimit -Ss 10240` - -- Used to silence IBM MCM warnings - posix.setrlimit ("stack", 10485760) -end - --- vi: ts=4 sw=4 expandtab diff --git a/src/modules/wreck/lua.d/timeout.lua b/src/modules/wreck/lua.d/timeout.lua deleted file mode 100644 index 3e61bed34a30..000000000000 --- a/src/modules/wreck/lua.d/timeout.lua +++ /dev/null @@ -1,61 +0,0 @@ --- --- Register timer on nodeid 0 if kvs `walltime` is set for this job. --- Kill job on timeout if reached. --- -local posix = require 'flux.posix' - -local function signal_to_number (s) - if not s then return nil end - local ret = tonumber (s) - if ret then return ret end - return posix [s] -end - -local function timeout_signal (f, wreck) - -- If 'lwj.walltime-signal' set then return this signal number, - -- otherwise if 'lwj.walltime-signal' set use this number, - -- otherwise return default signal 'SIGALRM' - -- - local s = signal_to_number (wreck.kvsdir ['walltime-signal']) - if s then return s end - local s = signal_to_number (f:kvsdir() ["lwj.walltime-signal"]) - if s then return s end - return posix.SIGALRM -end - -local function start_timer (f, wreck, id, walltime) - local to, err = f:timer { - timeout = walltime*1000, - oneshot = true, - handler = function (f, to) - local signum = timeout_signal (f, wreck) - wreck:log_error ("Timeout expired! Killing job with signal %d", signum) - local rc,err = f:sendevent ({signal = signum}, "wreck.%d.kill", id) - if not rc then - wreck:log_msg ("Failed to send kill event: %s", err) - end - end - } -end - -function rexecd_init () - if wreck.nodeid ~= 0 then return end - - local walltime = tonumber (wreck.kvsdir.walltime) - if not walltime or walltime <= 0 then return end - - local id = wreck.id - local f = wreck.flux - -- start timer when state is running - local k, err = f:kvswatcher { - key = tostring (wreck.kvsdir) .. ".state", - handler = function (k, result) - if result == "running" then - wreck:log_msg ("detected job state = running, starting timer for %ds", walltime) - k:remove() - start_timer (f, wreck, id, walltime) - end - end - } -end - diff --git a/src/modules/wreck/luastack.c b/src/modules/wreck/luastack.c deleted file mode 100644 index d8a76a0cad96..000000000000 --- a/src/modules/wreck/luastack.c +++ /dev/null @@ -1,455 +0,0 @@ -/************************************************************\ - * Copyright 2014 Lawrence Livermore National Security, LLC - * (c.f. AUTHORS, NOTICE.LLNS, COPYING) - * - * This file is part of the Flux resource manager framework. - * For details, see https://github.com/flux-framework. - * - * SPDX-License-Identifier: LGPL-3.0 -\************************************************************/ - -#if HAVE_CONFIG_H -#include "config.h" -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "list.h" -#include "luastack.h" - -/**************************************************************************** - * Datatypes - ****************************************************************************/ - -#define LUA_SCRIPT_FILE 1 -#define LUA_SCRIPT_STRING 2 - -struct lua_script { - int type; /* Type of loaded script, a file or a raw string */ - char * data; /* Data: File path or string */ - char * label;/* Filename or name of this script (for errors) */ - - lua_stack_t st; /* Pointer back to lua_stack in which we're loaded */ - lua_State * L; /* Copy of Lua state */ - int env_ref; /* Reference for _ENV table (5.1 setfenv table) */ -}; - -struct lua_script_stack { - lua_State *L; /* Global lua state */ - l_err_f errf; - List script_list; /* List of scripts in this stack */ -}; - - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static int lua_script_is_file (lua_script_t s) -{ - return (s->type == LUA_SCRIPT_FILE); -} - -static int lua_script_is_string (lua_script_t s) -{ - return (s->type == LUA_SCRIPT_STRING); -} - -static void lua_script_destroy (struct lua_script *s) -{ - if (s == NULL) - return; - if (s->L && s->st) { - luaL_unref (s->L, LUA_REGISTRYINDEX, s->env_ref); - /* Only call lua_close() on global/main lua state */ - s->st = NULL; - s->L = NULL; - } - s->type = 0; - free (s->data); - free (s->label); - free (s); -} - - -lua_script_t lua_script_create (lua_stack_t st, int type, const char *data) -{ - struct lua_script *script; - struct lua_State *L; - - if (!st || !st->L) - return (NULL); - - if (type != LUA_SCRIPT_FILE && type != LUA_SCRIPT_STRING) { - (*st->errf) ("lua_script_create: Invalid type!"); - return (NULL); - } - - if ((script = malloc (sizeof (*script))) == NULL) - return (NULL); - - memset (script, 0, sizeof (*script)); - - script->type = type; - if (!(script->data = strdup (data))) { - lua_script_destroy (script); - return (NULL); - } - - if (type == LUA_SCRIPT_FILE && - !(script->label = strdup (basename (script->data)))) { - lua_script_destroy (script); - return (NULL); - } - - L = st->L; - script->st = st; - script->L = L; - - /* New globals table/_ENV for this chunk */ - lua_newtable (script->L); - - /* metatable for table on top of stack */ - lua_newtable (script->L); - - /* - * Now set metatable->__index to point to the real globals - * table. This way Lua will check the root global table - * for any nonexistent items in the current chunk's environment - * table. - */ - lua_pushstring (script->L, "__index"); - lua_getglobal (script->L, "_G"); - lua_settable (script->L, -3); - - /* Now set metatable for the new globals table */ - lua_setmetatable (script->L, -2); - - /* Save reference to this table, which will be used as _ENV for loaded chunk */ - script->env_ref = luaL_ref (script->L, LUA_REGISTRYINDEX); - return (script); -} - -static inline void print_lua_script_error (lua_stack_t st, lua_script_t s) -{ - (*st->errf) ("%s: %s\n", s->label, lua_tostring (s->L, -1)); -} - -static int lua_script_compile (lua_stack_t st, lua_script_t s) -{ - /* - * First load Lua script (via loadfile or loadbuffer) - */ - if (lua_script_is_file (s) && luaL_loadfile (s->L, s->data)) { - (*st->errf) ("%s: Script failed to load.\n", s->data); - return (-1); - } - else if (lua_script_is_string (s) && - luaL_loadbuffer (s->L, s->data, strlen (s->data), s->label)) { - (*st->errf) ("%s: Failed to load script.\n", s->data); - return (-1); - } - - /* Get the environment/globals table for this script from - * the registry and set it as globals table for this chunk - */ - lua_rawgeti (s->L, LUA_REGISTRYINDEX, s->env_ref); -#if LUA_VERSION_NUM >= 502 - /* 5.2 and greater: set table as first upvalue: i.e. _ENV */ - lua_setupvalue (s->L, -2, 1); -#else - /* 5.0, 5.1: Set table as function environment for the chunk */ - lua_setfenv (s->L, -2); -#endif - - /* - * Now compile the loaded script: - */ - if (lua_pcall (s->L, 0, 0, 0)) { - print_lua_script_error (st, s); - return (-1); - } - - return (0); -} - -static int ef (const char *p, int eerrno) -{ - /* FIXME */ - //fprintf (stderr, "glob: %s: %s\n", p, strerror (eerrno)); - return (-1); -} - -int lua_script_list_append (lua_stack_t st, const char *pattern) -{ - glob_t gl; - size_t i; - int rc; - int type = LUA_SCRIPT_FILE; - - if (pattern == NULL || st == NULL || st->script_list == NULL) - return (-1); - - rc = glob (pattern, GLOB_ERR, ef, &gl); - switch (rc) { - case 0: - for (i = 0; i < gl.gl_pathc; i++) { - struct lua_script *s; - if (!(s = lua_script_create (st, type, gl.gl_pathv[i])) || - (lua_script_compile (st, s) < 0)) { - (*st->errf) ("%s: Failed. Skipping.\n", gl.gl_pathv[i]); - lua_script_destroy (s); - continue; - } - list_append (st->script_list, s); - } - break; - case GLOB_NOMATCH: - break; - case GLOB_NOSPACE: - (*st->errf) ("glob: Out of memory\n"); - break; - case GLOB_ABORTED: - (*st->errf) ("Cannot read %s: %s\n", pattern, strerror (errno)); - break; - default: - (*st->errf) ("Unknown glob rc = %d\n", rc); - break; - } - - globfree (&gl); - return (0); -} - -static int lua_script_rc (lua_State *L) -{ - return (lua_isnumber (L, -1) ? lua_tonumber (L, -1) : 0); -} - -static void verr (const char *fmt, ...) -{ - va_list ap; - va_start (ap, fmt); - vfprintf (stderr, fmt, ap); - va_end (ap); -} - -void lua_stack_destroy (lua_stack_t st) -{ - if (st == NULL) - return; - if (st->script_list) - list_destroy (st->script_list); - if (st->L) - lua_close (st->L); - free (st); -} - -lua_stack_t lua_stack_create (void) -{ - lua_stack_t s = malloc (sizeof (*s)); - - if (s == NULL) - return (NULL); - - if (!(s->script_list = list_create ((ListDelF) lua_script_destroy))) { - free (s); - return (NULL); - } - - s->L = luaL_newstate (); - s->errf = &verr; - - luaL_openlibs(s->L); - - return (s); -} - -int lua_stack_append_file (lua_stack_t st, const char *pattern) -{ - if (st == NULL || st->script_list == NULL) - return (-1); - - if (lua_script_list_append (st, pattern) < 0) - return (-1); - - return (0); -} - -int lua_stack_append_script (lua_stack_t st, const char *script, - const char *label) -{ - int type = LUA_SCRIPT_STRING; - lua_script_t s = lua_script_create (st, type, script); - if (s == NULL) - return (-1); - - s->label = label ? strdup (label) : strdup ("