From 659fbba5f619f229f95a2c867cece5f787a5a743 Mon Sep 17 00:00:00 2001 From: singularitti Date: Sun, 11 Jun 2023 22:58:05 -0400 Subject: [PATCH] Move code EasyJobsBase.jl under EasyJobs.jl --- EasyJobsBase/LICENSE | 21 ++++ EasyJobsBase/Project.toml | 19 ++++ EasyJobsBase/README.md | 66 ++++++++++++ EasyJobsBase/src/EasyJobsBase.jl | 10 ++ EasyJobsBase/src/jobs.jl | 140 +++++++++++++++++++++++++ EasyJobsBase/src/misc.jl | 67 ++++++++++++ EasyJobsBase/src/operations.jl | 54 ++++++++++ EasyJobsBase/src/run.jl | 170 +++++++++++++++++++++++++++++++ EasyJobsBase/src/show.jl | 56 ++++++++++ EasyJobsBase/src/status.jl | 104 +++++++++++++++++++ EasyJobsBase/test/run.jl | 150 +++++++++++++++++++++++++++ EasyJobsBase/test/runtests.jl | 6 ++ 12 files changed, 863 insertions(+) create mode 100644 EasyJobsBase/LICENSE create mode 100644 EasyJobsBase/Project.toml create mode 100644 EasyJobsBase/README.md create mode 100644 EasyJobsBase/src/EasyJobsBase.jl create mode 100644 EasyJobsBase/src/jobs.jl create mode 100644 EasyJobsBase/src/misc.jl create mode 100644 EasyJobsBase/src/operations.jl create mode 100644 EasyJobsBase/src/run.jl create mode 100644 EasyJobsBase/src/show.jl create mode 100644 EasyJobsBase/src/status.jl create mode 100644 EasyJobsBase/test/run.jl create mode 100644 EasyJobsBase/test/runtests.jl diff --git a/EasyJobsBase/LICENSE b/EasyJobsBase/LICENSE new file mode 100644 index 0000000..ac60394 --- /dev/null +++ b/EasyJobsBase/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 singularitti and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/EasyJobsBase/Project.toml b/EasyJobsBase/Project.toml new file mode 100644 index 0000000..11e1be0 --- /dev/null +++ b/EasyJobsBase/Project.toml @@ -0,0 +1,19 @@ +name = "EasyJobsBase" +uuid = "db8ca866-b61f-4bd1-a9b9-75c107d645d4" +authors = ["singularitti and contributors"] +version = "0.8.0" + +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +Thinkers = "6d80a3f9-a943-41fa-97b3-3004c0daf7a3" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[compat] +Thinkers = "0.2" +julia = "1" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/EasyJobsBase/README.md b/EasyJobsBase/README.md new file mode 100644 index 0000000..4cbf0a0 --- /dev/null +++ b/EasyJobsBase/README.md @@ -0,0 +1,66 @@ +# EasyJobsBase + +| **Documentation** | **Build Status** | **Others** | +| :--------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------: | +| [![Stable][docs-stable-img]][docs-stable-url] [![Dev][docs-dev-img]][docs-dev-url] | [![Build Status][gha-img]][gha-url] [![Build Status][appveyor-img]][appveyor-url] [![Build Status][cirrus-img]][cirrus-url] [![pipeline status][gitlab-img]][gitlab-url] [![Coverage][codecov-img]][codecov-url] | [![GitHub license][license-img]][license-url] [![Code Style: Blue][style-img]][style-url] | + +[docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg +[docs-stable-url]: https://MineralsCloud.github.io/EasyJobsBase.jl/stable +[docs-dev-img]: https://img.shields.io/badge/docs-dev-blue.svg +[docs-dev-url]: https://MineralsCloud.github.io/EasyJobsBase.jl/dev +[gha-img]: https://github.com/MineralsCloud/EasyJobsBase.jl/workflows/CI/badge.svg +[gha-url]: https://github.com/MineralsCloud/EasyJobsBase.jl/actions +[appveyor-img]: https://ci.appveyor.com/api/projects/status/github/MineralsCloud/EasyJobsBase.jl?svg=true +[appveyor-url]: https://ci.appveyor.com/project/singularitti/EasyJobsBase-jl +[cirrus-img]: https://api.cirrus-ci.com/github/MineralsCloud/EasyJobsBase.jl.svg +[cirrus-url]: https://cirrus-ci.com/github/MineralsCloud/EasyJobsBase.jl +[gitlab-img]: https://gitlab.com/singularitti/EasyJobsBase.jl/badges/main/pipeline.svg +[gitlab-url]: https://gitlab.com/singularitti/EasyJobsBase.jl/-/pipelines +[codecov-img]: https://codecov.io/gh/MineralsCloud/EasyJobsBase.jl/branch/main/graph/badge.svg +[codecov-url]: https://codecov.io/gh/MineralsCloud/EasyJobsBase.jl +[license-img]: https://img.shields.io/github/license/MineralsCloud/EasyJobsBase.jl +[license-url]: https://github.com/MineralsCloud/EasyJobsBase.jl/blob/main/LICENSE +[style-img]: https://img.shields.io/badge/code%20style-blue-4495d1.svg +[style-url]: https://github.com/invenia/BlueStyle + +The code is [hosted on GitHub](https://github.com/MineralsCloud/EasyJobsBase.jl), +with some continuous integration services to test its validity. + +This repository is created and maintained by [@singularitti](https://github.com/singularitti). +You are very welcome to contribute. + +## Installation + +The package can be installed with the Julia package manager. +From the Julia REPL, type `]` to enter the Pkg REPL mode and run: + +``` +pkg> add EasyJobsBase +``` + +Or, equivalently, via the [`Pkg` API](https://pkgdocs.julialang.org/v1/getting-started/): + +```julia +julia> import Pkg; Pkg.add("EasyJobsBase") +``` + +## Documentation + +- [**STABLE**][docs-stable-url] — **documentation of the most recently tagged version.** +- [**DEV**][docs-dev-url] — _documentation of the in-development version._ + +## Project status + +The package is tested against, and being developed for, Julia `1.6` and above on Linux, +macOS, and Windows. + +## Questions and contributions + +You are welcome to post usage questions on [our discussion page][discussions-url]. + +Contributions are very welcome, as are feature requests and suggestions. Please open an +[issue][issues-url] if you encounter any problems. The [Contributing](@ref) page has +guidelines that should be followed when opening pull requests and contributing code. + +[discussions-url]: https://github.com/MineralsCloud/EasyJobsBase.jl/discussions +[issues-url]: https://github.com/MineralsCloud/EasyJobsBase.jl/issues diff --git a/EasyJobsBase/src/EasyJobsBase.jl b/EasyJobsBase/src/EasyJobsBase.jl new file mode 100644 index 0000000..cb7b614 --- /dev/null +++ b/EasyJobsBase/src/EasyJobsBase.jl @@ -0,0 +1,10 @@ +module EasyJobsBase + +include("jobs.jl") +include("operations.jl") +include("run.jl") +include("status.jl") +include("misc.jl") +include("show.jl") + +end diff --git a/EasyJobsBase/src/jobs.jl b/EasyJobsBase/src/jobs.jl new file mode 100644 index 0000000..d3fc3bc --- /dev/null +++ b/EasyJobsBase/src/jobs.jl @@ -0,0 +1,140 @@ +using Dates: DateTime, now +using UUIDs: UUID, uuid1 + +using Thinkers: Think + +export Job, WeaklyDependentJob, StronglyDependentJob + +@enum JobStatus begin + PENDING + RUNNING + SUCCEEDED + FAILED + INTERRUPTED +end + +abstract type AbstractJob end +# Reference: https://github.com/cihga39871/JobSchedulers.jl/blob/aca52de/src/jobs.jl#L35-L69 +""" + Job(core::Thunk; description="", username="") + +Create a simple job. + +# Arguments +- `core`: a `Thunk` that encloses the job core definition. +- `name`: give a short name to the job. +- `description::String=""`: describe what the job does in more detail. +- `username::String=""`: indicate who executes the job. + +# Examples +```jldoctest +julia> using Thinkers + +julia> a = Job(Thunk(sleep, 5); username="me", description="Sleep for 5 seconds"); + +julia> b = Job(Thunk(run, `pwd` & `ls`); username="me", description="Run some commands"); +``` +""" +mutable struct Job <: AbstractJob + id::UUID + core::Think + name::String + description::String + username::String + creation_time::DateTime + start_time::DateTime + end_time::DateTime + "Track the job status." + status::JobStatus + "Count hom many times the job has been run." + count::UInt64 + "These jobs runs before the current job." + parents::Set{AbstractJob} + "These jobs runs after the current job." + children::Set{AbstractJob} + function Job(core::Think; name="", description="", username="") + return new( + uuid1(), + core, + name, + description, + username, + now(), + DateTime(0), + DateTime(0), + PENDING, + 0, + Set(), + Set(), + ) + end +end +abstract type DependentJob <: AbstractJob end +mutable struct WeaklyDependentJob <: DependentJob + id::UUID + core::Think + name::String + description::String + username::String + creation_time::DateTime + start_time::DateTime + end_time::DateTime + "Track the job status." + status::JobStatus + "Count hom many times the job has been run." + count::UInt64 + "These jobs runs before the current job." + parents::Set{AbstractJob} + "These jobs runs after the current job." + children::Set{AbstractJob} + function WeaklyDependentJob(core::Think; name="", description="", username="") + return new( + uuid1(), + core, + name, + description, + username, + now(), + DateTime(0), + DateTime(0), + PENDING, + 0, + Set(), + Set(), + ) + end +end +mutable struct StronglyDependentJob <: DependentJob + id::UUID + core::Think + name::String + description::String + username::String + creation_time::DateTime + start_time::DateTime + end_time::DateTime + "Track the job status." + status::JobStatus + "Count hom many times the job has been run." + count::UInt64 + "These jobs runs before the current job." + parents::Set{AbstractJob} + "These jobs runs after the current job." + children::Set{AbstractJob} + function StronglyDependentJob(core::Think; name="", description="", username="") + return new( + uuid1(), + core, + name, + description, + username, + now(), + DateTime(0), + DateTime(0), + PENDING, + 0, + Set(), + Set(), + ) + end +end diff --git a/EasyJobsBase/src/misc.jl b/EasyJobsBase/src/misc.jl new file mode 100644 index 0000000..557bc07 --- /dev/null +++ b/EasyJobsBase/src/misc.jl @@ -0,0 +1,67 @@ +import Thinkers: getresult + +export countexecution, + descriptionof, creationtimeof, starttimeof, endtimeof, timecostof, getresult + +""" + countexecution(job::AbstractJob) + +Count how many times the `job` has been run. +""" +countexecution(job::AbstractJob) = Int(job.count) + +""" + descriptionof(job::AbstractJob) + +Return the description of the `job`. +""" +descriptionof(job::AbstractJob) = job.description + +""" + creationtimeof(job::AbstractJob) + +Return the creation time of the `job`. +""" +creationtimeof(job::AbstractJob) = job.creation_time + +""" + starttimeof(job::AbstractJob) + +Return the start time of the `job`. Return `nothing` if it is still pending. +""" +starttimeof(job::AbstractJob) = ispending(job) ? nothing : job.start_time + +""" + endtimeof(job::AbstractJob) + +Return the end time of the `job`. Return `nothing` if it has not exited. +""" +endtimeof(job::AbstractJob) = isexited(job) ? job.end_time : nothing + +""" + timecostof(job::AbstractJob) + +Return the time cost of the `job` since it started running. + +If `nothing`, the `job` is still pending. If it is finished, return how long it took to +complete. +""" +function timecostof(job::AbstractJob) + if ispending(job) + return nothing + elseif isrunning(job) + return now() - job.start_time + else # Exited + return job.end_time - job.start_time + end +end + +""" + getresult(job::AbstractJob) + +Get the running result of the `job`. + +The result is wrapped by a `Some` type. Use `something` to retrieve its value. +If it is `nothing`, the `job` is not finished. +""" +getresult(job::AbstractJob) = isexited(job) ? getresult(job.core) : nothing diff --git a/EasyJobsBase/src/operations.jl b/EasyJobsBase/src/operations.jl new file mode 100644 index 0000000..acf578f --- /dev/null +++ b/EasyJobsBase/src/operations.jl @@ -0,0 +1,54 @@ +export chain!, →, ← + +""" + chain!(x::AbstractJob, y::AbstractJob, z::AbstractJob...) + +Chain multiple `AbstractJob`s one after another. +""" +function chain!(x::AbstractJob, y::AbstractJob) + if x == y + throw(ArgumentError("a job cannot be followed by itself!")) + end + if y in x.children && x in y.parents + @info "You cannot chain the same jobs twice! No operations will be done!" + elseif y in x.children || x in y.parents # This should never happen + error("Only one job is linked to the other, something is wrong!") + else + push!(x.children, y) + push!(y.parents, x) + end + return x +end +chain!(x::AbstractJob, y::AbstractJob, z::AbstractJob...) = foldr(chain!, (x, y, z...)) +""" + →(x, y) + +Chain two `AbstractJob`s. +""" +→(x::AbstractJob, y::AbstractJob) = chain!(x, y) +""" + ←(y, x) + +Chain two `AbstractJob`s reversely. +""" +←(y::AbstractJob, x::AbstractJob) = x → y + +# See https://github.com/JuliaLang/julia/blob/70c873e/base/number.jl#L279-L280 +Base.iterate(x::AbstractJob) = (x, nothing) +Base.iterate(::AbstractJob, ::Any) = nothing + +# See https://github.com/JuliaLang/julia/blob/70c873e/base/number.jl#L92 +Base.IteratorSize(::Type{<:AbstractJob}) = Base.HasShape{0}() + +# See https://github.com/JuliaLang/julia/blob/70c873e/base/number.jl#L84 +Base.eltype(::Type{T}) where {T<:AbstractJob} = T + +# See https://github.com/JuliaLang/julia/blob/70c873e/base/number.jl#L87 +Base.length(::AbstractJob) = 1 + +# See https://github.com/JuliaLang/julia/blob/70c873e/base/number.jl#L80-L81 +Base.size(::AbstractJob) = () +Base.size(::AbstractJob, dim::Integer) = dim < 1 ? throw(BoundsError()) : 1 + +# See https://github.com/JuliaLang/julia/blob/70c873e/base/number.jl#L282 +Base.in(x::AbstractJob, y::AbstractJob) = x == y diff --git a/EasyJobsBase/src/run.jl b/EasyJobsBase/src/run.jl new file mode 100644 index 0000000..4c9a9ca --- /dev/null +++ b/EasyJobsBase/src/run.jl @@ -0,0 +1,170 @@ +using Thinkers: TimeoutException, ErrorInfo, reify!, setargs!, haserred, _kill + +export run!, execute!, kill! + +# See https://github.com/MineralsCloud/SimpleWorkflows.jl/issues/137 +""" + Executor(job::AbstractJob; wait=false, maxattempts=1, interval=1, delay=0) + +Handle the execution of jobs. + +# Arguments +- `job::AbstractJob`: an `AbstractJob` instance. +- `wait::Bool=false`: determines whether to wait for the job to complete before executing the next task. +- `maxattempts::UInt64=1`: the maximum number of attempts to execute the job. +- `interval::Real=1`: the time interval between each attempt to execute the job, in seconds. +- `delay::Real=0`: the delay before the first attempt to execute the job, in seconds. +""" +mutable struct Executor{T<:AbstractJob} + job::T + wait::Bool + maxattempts::UInt64 + interval::Real + delay::Real + task::Task + function Executor(job::T; wait=false, maxattempts=1, interval=1, delay=0) where {T} + @assert maxattempts >= 1 + @assert interval >= zero(interval) + @assert delay >= zero(delay) + return new{T}(job, wait, maxattempts, interval, delay, @task _run!(job)) + end +end + +function newtask!(exec::Executor) + exec.task = @task _run!(exec.job) # Start a new task. This is necessary for rerunning! + return exec +end + +""" + run!(job::Job; wait=false, maxattempts=1, interval=1, delay=0) + +Run a `Job` with a maximum number of attempts, with each attempt separated by `interval` seconds +and an initial `delay` in seconds. +""" +function run!(job::AbstractJob; kwargs...) + exec = Executor(job; kwargs...) + execute!(exec) + return exec +end + +""" + execute!(exec::Executor) + +Execute a given job associated with the `Executor` object. + +# Arguments +- `exec::Executor`: the `Executor` object containing the job to be executed. +""" +function execute!(exec::Executor) + @assert shouldrun(exec) + prepare!(exec) + return launch!(exec) +end + +""" + launch!(exec::Executor) + +Internal function to execute a given job associated with the `Executor` object. + +This function checks if the job has succeeded. If not, it sleeps for a delay, +runs the job once using `singlerun!`. If `maxattempts` is more than ``1``, it loops over +the remaining attempts, sleeping for an interval, running the job, and waiting in each loop. +If the job has already succeeded, it stops immediately. +""" +function launch!(exec::Executor) # Do not export! + if !issucceeded(exec.job) + sleep(exec.delay) + singlerun!(exec) # Wait or not depends on `exec.wait` + if exec.maxattempts > 1 + wait(exec) + for _ in Base.OneTo(exec.maxattempts - 1) + sleep(exec.interval) + singlerun!(exec) + wait(exec) # Wait no matter whether `exec.wait` is `true` or `false` + end + end + end + return exec # Stop immediately if the job has succeeded +end + +""" + singlerun!(exec::Executor) + +Executes a single run of the job associated with the `Executor` object. + +This function checks the job status. If the job is pending, it schedules the task and waits +if `wait` is `true`. If the job has failed or been interrupted, it creates a new task, +resets the job status to `PENDING`, and then calls `singlerun!` again. If the job is running +or has succeeded, it does nothing and returns the `Executor` object. +""" +function singlerun!(exec::Executor) + if ispending(exec.job) + schedule(exec.task) + if exec.wait + wait(exec) + end + end + if isfailed(exec.job) || isinterrupted(exec.job) + newtask!(exec) + exec.job.status = PENDING + return singlerun!(exec) # Wait or not depends on `exec.wait` + end + return exec # Do nothing for running and succeeded jobs +end + +# Internal function to execute a specific `AbstractJob`. +function _run!(job::AbstractJob) # Do not export! + job.status = RUNNING + job.start_time = now() + reify!(job.core) + job.end_time = now() + job.status = if haserred(job.core) + e = something(getresult(job.core)).thrown + e isa Union{InterruptException,TimeoutException} ? INTERRUPTED : FAILED + else + SUCCEEDED + end + job.count += 1 + return job +end + +prepare!(::Executor) = nothing # No op +function prepare!(exec::Executor{StronglyDependentJob}) + parents = exec.job.parents + # Use previous results as arguments + args = if length(parents) == 1 + something(getresult(first(parents))) + else # > 1 + Set(something(getresult(parent)) for parent in parents) + end + setargs!(exec.job.core, args) + return nothing +end + +shouldrun(::Executor) = true +shouldrun(exec::Executor{<:DependentJob}) = + length(exec.job.parents) >= 1 && all(issucceeded(parent) for parent in exec.job.parents) + +""" + kill!(exec::Executor) + +Manually kill a `Job`, works only if it is running. +""" +function kill!(exec::Executor) + if isexited(exec.job) + @info "the job $(exec.job.id) has already exited!" + elseif ispending(exec.job) + @info "the job $(exec.job.id) has not started!" + else + _kill(exec.task) + end + return exec +end + +""" + Base.wait(exec::Executor) + +Overloads the Base `wait` function to wait for the `Task` associated with an `Executor` +object to complete. +""" +Base.wait(exec::Executor) = wait(exec.task) diff --git a/EasyJobsBase/src/show.jl b/EasyJobsBase/src/show.jl new file mode 100644 index 0000000..82f3da0 --- /dev/null +++ b/EasyJobsBase/src/show.jl @@ -0,0 +1,56 @@ +using Dates: format + +# See https://docs.julialang.org/en/v1/manual/types/#man-custom-pretty-printing +function Base.show(io::IO, job::AbstractJob) + if get(io, :compact, false) + print( + IOContext(io, :limit => true, :compact => true), summary(job), '(', job.id, ')' + ) + else + print(io, summary(job), '(', job.id, "), ", job.core) + end +end +function Base.show(io::IO, ::MIME"text/plain", job::AbstractJob) + println(io, summary(job)) + println(io, ' ', "id: ", job.id) + if !isempty(job.description) + print(io, ' ', "description: ") + show(io, job.description) + println(io) + end + print(io, ' ', "core: ") + printf(io, job.core) + print(io, '\n', ' ', "status: ") + printstyled(io, getstatus(job); bold=true) + if !ispending(job) + println(io, '\n', ' ', "from: ", format(starttimeof(job), "HH:MM:SS.s, mm/dd/yyyy")) + if isrunning(job) + print(io, " still running...") + else + println(io, ' ', "to: ", format(endtimeof(job), "HH:MM:SS.s, mm/dd/yyyy")) + print(io, ' ', "used: ", timecostof(job)) + end + end +end + +function printf(io::IO, think::Think) + print(io, think.callable, '(') + args = think.args + if length(args) > 0 + for v in args[1:(end - 1)] + print(io, v, ", ") + end + print(io, args[end]) + end + kwargs = think.kwargs + if isempty(kwargs) + print(io, ')') + else + print(io, ";") + for (k, v) in zip(keys(kwargs)[1:(end - 1)], Tuple(kwargs)[1:(end - 1)]) + print(io, ' ', k, '=', v, ",") + end + print(io, ' ', last(keys(kwargs)), '=', last(values(kwargs))) + print(io, ')') + end +end diff --git a/EasyJobsBase/src/status.jl b/EasyJobsBase/src/status.jl new file mode 100644 index 0000000..37fc6b9 --- /dev/null +++ b/EasyJobsBase/src/status.jl @@ -0,0 +1,104 @@ +export getstatus, + ispending, + isrunning, + isexited, + issucceeded, + isfailed, + isinterrupted, + listpending, + listrunning, + listexited, + listsucceeded, + listfailed, + listinterrupted + +""" + getstatus(job::AbstractJob) + +Get the current status of the `job`. +""" +getstatus(job::AbstractJob) = job.status + +""" + ispending(job::AbstractJob) + +Test if the `job` is still pending. +""" +ispending(job::AbstractJob) = getstatus(job) === PENDING + +""" + isrunning(job::AbstractJob) + +Test if the `job` is running. +""" +isrunning(job::AbstractJob) = getstatus(job) === RUNNING + +""" + isexited(job::AbstractJob) + +Test if the `job` has exited. +""" +isexited(job::AbstractJob) = getstatus(job) in (SUCCEEDED, FAILED, INTERRUPTED) + +""" + issucceeded(job::AbstractJob) + +Test if the `job` was successfully run. +""" +issucceeded(job::AbstractJob) = getstatus(job) === SUCCEEDED + +""" + isfailed(job::AbstractJob) + +Test if the `job` failed during running. +""" +isfailed(job::AbstractJob) = getstatus(job) === FAILED + +""" + isinterrupted(job::AbstractJob) + +Test if the `job` was interrupted during running. +""" +isinterrupted(job::AbstractJob) = getstatus(job) === INTERRUPTED + +""" + listpending(jobs) + +Filter the pending jobs in a sequence of jobs. +""" +listpending(jobs) = Iterators.filter(ispending, jobs) + +""" + listrunning(jobs) + +Filter the running jobs in a sequence of jobs. +""" +listrunning(jobs) = Iterators.filter(isrunning, jobs) + +""" + listexited(jobs) + +Filter the exited jobs in a sequence of jobs. +""" +listexited(jobs) = Iterators.filter(isexited, jobs) + +""" + listsucceeded(jobs) + +Filter the succeeded jobs in a sequence of jobs. +""" +listsucceeded(jobs) = Iterators.filter(issucceeded, jobs) + +""" + listfailed(jobs) + +Filter the failed jobs in a sequence of jobs. +""" +listfailed(jobs) = Iterators.filter(isfailed, jobs) + +""" + listinterrupted(jobs) + +Filter the interrupted jobs in a sequence of jobs. +""" +listinterrupted(jobs) = Iterators.filter(isinterrupted, jobs) diff --git a/EasyJobsBase/test/run.jl b/EasyJobsBase/test/run.jl new file mode 100644 index 0000000..6776996 --- /dev/null +++ b/EasyJobsBase/test/run.jl @@ -0,0 +1,150 @@ +using Thinkers + +@testset "Test running a `Job` multiple times" begin + function f() + n = rand(1:5) + n < 5 ? error("not the number we want!") : return n + end + i = Job(Thunk(f); username="me", name="i") + run!(i; maxattempts=10, interval=3) + count = countexecution(i) + @test 1 <= count <= 10 + run!(i; maxattempts=10, interval=3) + @test countexecution(i) == count +end + +@testset "Test running `Job`s" begin + function f₁() + println("Start job `i`!") + return sleep(5) + end + function f₂(n) + println("Start job `j`!") + sleep(n) + return exp(2) + end + function f₃(n) + println("Start job `k`!") + return sleep(n) + end + function f₄() + println("Start job `l`!") + return run(`sleep 3`) + end + function f₅(n, x) + println("Start job `m`!") + sleep(n) + return sin(x) + end + function f₆(n; x=1) + println("Start job `n`!") + sleep(n) + cos(x) + return run(`pwd` & `ls`) + end + @testset "No dependency" begin + i = Job(Thunk(f₁); username="me", name="i") + j = Job(Thunk(f₂, 3); username="he", name="j") + k = Job(Thunk(f₃, 6); name="k") + l = Job(Thunk(f₄); name="l", username="me") + m = Job(Thunk(f₅, 3, 1); name="m") + n = Job(Thunk(f₆, 1; x=3); username="she", name="n") + for job in (i, j, k, l, m, n) + exec = run!(job) + wait(exec) + @test issucceeded(job) + end + end + @testset "Related jobs" begin + i = Job(Thunk(f₁); username="me", name="i") + j = Job(Thunk(f₂, 3); username="he", name="j") + k = Job(Thunk(f₃, 6); name="k") + l = Job(Thunk(f₄); name="l", username="me") + m = Job(Thunk(f₅, 3, 1); name="m") + n = Job(Thunk(f₆, 1; x=3); username="she", name="n") + i .→ [j, k] .→ [l, m] .→ n + @assert isempty(i.parents) + @assert i.children == Set([j, k]) + @assert j.parents == Set([i]) + @assert j.children == Set([l]) + @assert k.parents == Set([i]) + @assert k.children == Set([m]) + @assert l.parents == Set([j]) + @assert l.children == Set([n]) + @assert m.parents == Set([k]) + @assert m.children == Set([n]) + @assert n.parents == Set([l, m]) + @assert isempty(n.children) + for job in (i, j, k, l, m, n) + exec = run!(job) + wait(exec) + @test issucceeded(job) + end + end +end + +@testset "Test running `WeaklyDependentJob`s" begin + f₁(x) = write("file", string(x)) + f₂() = read("file", String) + h = Job(Thunk(sleep, 3); username="me", name="h") + i = Job(Thunk(f₁, 1001); username="me", name="i") + j = WeaklyDependentJob(Thunk(map, f₂); username="he", name="j") + [h, i] .→ j + @test_throws AssertionError run!(j) + @test getresult(j) === nothing + exec = run!(h) + wait(exec) + @test_throws AssertionError run!(j) + @test getresult(j) === nothing + exec = run!(i) + wait(exec) + exec = run!(j) + wait(exec) + @test getresult(j) == Some("1001") +end + +@testset "Test running `StronglyDependentJob`s" begin + f₁(x) = x^2 + f₂(y) = y + 1 + f₃(z) = z / 2 + i = Job(Thunk(f₁, 5); username="me", name="i") + j = StronglyDependentJob(Thunk(f₂, 3); username="he", name="j") + k = StronglyDependentJob(Thunk(f₃, 6); username="she", name="k") + i → j → k + @test_throws AssertionError run!(j) + exec = run!(i) + wait(exec) + @test getresult(i) == Some(25) + @test_throws AssertionError run!(k) + exec = run!(j) + wait(exec) + @test getresult(j) == Some(26) + exec = run!(k) + wait(exec) + @test getresult(k) == Some(13.0) +end + +@testset "Test running a `StronglyDependentJob` with more than one parent" begin + f₁(x) = x^2 + f₂(y) = y + 1 + f₃(z) = z / 2 + f₄(iter) = sum(iter) + i = Job(Thunk(f₁, 5); username="me", name="i") + j = Job(Thunk(f₂, 3); username="he", name="j") + k = Job(Thunk(f₃, 6); username="she", name="k") + l = StronglyDependentJob(Thunk(f₄, ()); username="she", name="me") + (i, j, k) .→ l + @test_throws AssertionError run!(l) + execs = map((i, j, k)) do job + run!(job) + end + for exec in execs + wait(exec) + end + exec = run!(l) + wait(exec) + @test getresult(i) == Some(25) + @test getresult(j) == Some(4) + @test getresult(k) == Some(3.0) + @test getresult(l) == Some(32.0) +end diff --git a/EasyJobsBase/test/runtests.jl b/EasyJobsBase/test/runtests.jl new file mode 100644 index 0000000..7bae4dd --- /dev/null +++ b/EasyJobsBase/test/runtests.jl @@ -0,0 +1,6 @@ +using EasyJobsBase +using Test + +@testset "EasyJobsBase.jl" begin + include("run.jl") +end