From f2f771e0cc6d5102ea9de9a50459db24d1a4f52f Mon Sep 17 00:00:00 2001 From: Morten Piibeleht Date: Tue, 2 Jul 2019 14:50:57 +1200 Subject: [PATCH] Update doctest API doctest() can now also doctest manual pages which makes it easier to include doctesting as part of the test suite. --- docs/src/man/doctests.md | 71 +++++++++++++------------------------ docs/src/showcase.md | 33 +++++++++++++++++ src/Documenter.jl | 63 ++++++++++++++++++++++++++++---- test/doctests/doctestapi.jl | 18 +++++----- test/manual.jl | 22 ++++++++++++ test/runtests.jl | 5 +-- 6 files changed, 147 insertions(+), 65 deletions(-) create mode 100644 docs/src/showcase.md create mode 100644 test/manual.jl diff --git a/docs/src/man/doctests.md b/docs/src/man/doctests.md index 6eff59eeeb8..58ca7da0209 100644 --- a/docs/src/man/doctests.md +++ b/docs/src/man/doctests.md @@ -311,67 +311,46 @@ julia> @time [1,2,3,4] keyword argument are all applied to each doctest. -## Doctesting Without Building the Docs +## Doctesting as Part of Testing -Documenter has a few ways to verify the doctests without having to run a potentially -expensive full documentation build. +Documenter provides the [`doctest`](@ref) function which can be used to verify all doctests +independently of manual builds. It simply returns `true` if the doctests pass, or `false` +if they do not. -### Doctesting docstrings only - -An option for doctesting just the docstrings of a particular module (and all its submodules) -is to use the [`doctest`](@ref) function. It takes a list of modules as an argument and runs -all the doctests in all the docstrings it finds. This can be handy for quick tests when -writing docstrings of a package. - -[`doctest`](@ref) will return `true` or `false`, depending on whether the doctests pass or -not, making it easy to include a doctest of all the docstrings in the package test suite: - -```julia -using MyPackage, Documenter, Test -@test doctest([MyPackage]) -``` - -Note that you still need to make sure that all the necessary [Module-level metadata](@ref) -for the doctests is set up before [`doctest`](@ref) is called. - -### Doctesting without a full build - -An alternative, which also runs doctests on the manual pages, but still skips most other -build steps, is to pass `doctest = :only` to [`makedocs`](@ref). - -This also makes it more practical to include doctests as part of the normal test suite of a -package. One option to set it up is to make the `doctest` keyword depend on command line -arguments passed to the `make.jl` script: +For example, it can be used to verify doctests as part of the normal test suite by having +e.g. the following in `runtests.jl`: ```julia -makedocs(..., - doctest = ("doctest-only" in ARGS) ? :only : true -) +using Test, Documenter, MyPackage +@testset "Doctesting" begin + @test doctest(MyPackage) +end ``` -Now, the `make.jl` script can be run on the command line as `julia docs/make.jl -doctest-only` and it will only run the doctests. On doctest failure, the `makedocs` throws -an error and `julia` exits with a non-zero exit code. +By default, it will also attempt to verify all the doctests on manual `.md` files, which it +assumes are located under `docs/src`. This can be configured or disabled with the `manual` +keyword (see [`doctest`](@ref) for more information). -For running the doctests as part of the standard test suite, the `docs/make.jl` can simply -be `include`d in the `test/runtest.jl` file: +For example, to test a package that does have separate manual pages, just docstrings, +`runtests.jl` can be set up as follows: ```julia -push!(ARGS, "doctest-only") -include(joinpath(@__DIR__, "..", "docs", "make.jl")) +using Test, Documenter, MyPackage +@testset "Doctesting" begin + @test doctest(MyPackage; manual = false) +end ``` -The `push!` to `ARGS` emulates the passing of the `doctest-only` command line argument. - -Note that, for this to work, you need to add Documenter and all the other packages that get -loaded in `make.jl`, or in the doctest, as test dependencies. +Note that you still need to make sure that all the necessary [Module-level metadata](@ref) +for the doctests is set up before [`doctest`](@ref) is called. Also, you need to add +Documenter and all the other packages you are loading in the doctests as test dependencies. ## Fixing Outdated Doctests -To fix outdated doctests, the `doctest` flag to [`makedocs`](@ref) can be set to -`doctest = :fix`. This will run the doctests, and overwrite the old results with -the new output. +To fix outdated doctests, the [`doctest`](@ref) function can be called with `fix = true`, +the `doctest = :fix` keyword can be passed to [`makedocs`](@ref). This will run the +doctests, and overwrite the old results with the new output. !!! note diff --git a/docs/src/showcase.md b/docs/src/showcase.md new file mode 100644 index 00000000000..5a584fa85a9 --- /dev/null +++ b/docs/src/showcase.md @@ -0,0 +1,33 @@ +# Hidden showcase page + +Currently exists just so that there would be doctests to run in manual pages of Documenter's +manual. This page does not show up in navigation. + +```jldoctest +julia> 2 + 2 +4 +``` + +The following doctests needs doctestsetup: + +```jldoctest; setup=:(using Documenter) +julia> Documenter.Utilities.splitexpr(:(Foo.Bar.baz)) +(:(Foo.Bar), :(:baz)) +``` + +Let's also try `@meta` blocks: + +```@meta +DocTestSetup = quote + f(x) = x^2 +end +``` + +```jldoctest +julia> f(2) +4 +``` + +```@meta +DocTestSetup = nothing +``` diff --git a/src/Documenter.jl b/src/Documenter.jl index f8944f6ebea..dc51da9053c 100644 --- a/src/Documenter.jl +++ b/src/Documenter.jl @@ -826,20 +826,71 @@ function getenv(regex::Regex) end """ - doctest(modules::AbstractVector{Module}) -> Bool + doctest(package::Module; kwargs...) -> Bool + +Convenience method that runs and checks all the doctests for a given Julia package. +`package` must be the `Module` object corresponding to the top-level module of the package. +Returns `true` if the doctesting was successful and false if any error occurred. + +# Keywords + +**`manual`** controls how manual pages are handled. By default (`manual = true`), `doctest` +assumes that manual pages are located under `docs/src`. If that is not the case, the +`manual` keyword argument can be passed to specify the directory. Setting `manual = false` +will skip doctesting of manual pages altogether. + +Additional keywords are passed on to the main [`doctest`](@ref) method. +""" +function doctest(package::Module; manual=true, kwargs...) + if pathof(package) === nothing + throw(ArgumentError("$(package) is not a top-level package module.")) + end + source = nothing + if manual === true + source = normpath(joinpath(dirname(pathof(package)), "..", "docs", "src")) + isdir(source) || throw(ArgumentError(""" + Package $(package) does not have a documentation source directory at standard location. + Searched at: $(source) + If ... + """)) + end + doctest(source, [package]; kwargs...) +end -Runs all the doctests in the given modules. Returns `true` if the doctesting was successful -and false if any error occurred. """ -function doctest(modules::AbstractVector{Module}) + doctest(source, modules; kwargs...) -> Bool + +Runs all the doctests in the given modules and on manual pages under the `source` directory. +Returns `true` if the doctesting was successful and `false` if any error occurred. + +The manual pages are searched recursively in subdirectories of `source` too. Doctesting of +manual pages can be disabled if `source` is set to `nothing`. + +# Keywords + +**`fix`**, if set to `true`, updates all the doctests that fail with the correct output +(default `false`). + +!!! warning + When running `doctest(...; fix=true)`, Documenter will modify the Markdown and Julia + source files. It is strongly recommended that you only run it on packages in Pkg's + develop mode and commit any staged changes. You should also review all the changes made + by `doctest` before committing them, as there may be edge cases when the automatic + fixing fails. +""" +function doctest(source::Union{AbstractString,Nothing}, modules::AbstractVector{Module}; fix=false) dir = mktempdir() @debug "Doctesting in temporary directory: $(dir)" modules - mkdir(joinpath(dir, "src")) + if source === nothing + source = joinpath(dir, "src") + mkdir(source) + end r = try makedocs( root = dir, + source = source, sitename = "", - doctest = :only, + doctest = fix ? :fix : :only, modules = modules, ) true diff --git a/test/doctests/doctestapi.jl b/test/doctests/doctestapi.jl index 923cfaab3c4..c3e7cc43a96 100644 --- a/test/doctests/doctestapi.jl +++ b/test/doctests/doctestapi.jl @@ -96,44 +96,44 @@ end @testset "Documenter.doctest" begin # DocTest1 - run_doctest([DocTest1]) do result, success, backtrace, output + run_doctest(nothing, [DocTest1]) do result, success, backtrace, output @test result end # DocTest2 - run_doctest([DocTest2]) do result, success, backtrace, output + run_doctest(nothing, [DocTest2]) do result, success, backtrace, output @test !result end # DocTest3 - run_doctest([DocTest3]) do result, success, backtrace, output + run_doctest(nothing, [DocTest3]) do result, success, backtrace, output @test !result end DocMeta.setdocmeta!(DocTest3, :DocTestSetup, :(x = 42)) - run_doctest([DocTest3]) do result, success, backtrace, output + run_doctest(nothing, [DocTest3]) do result, success, backtrace, output @test result end # DocTest4 - run_doctest([DocTest4]) do result, success, backtrace, output + run_doctest(nothing, [DocTest4]) do result, success, backtrace, output @test !result end DocMeta.setdocmeta!(DocTest4, :DocTestSetup, :(x = 42)) - run_doctest([DocTest4]) do result, success, backtrace, output + run_doctest(nothing, [DocTest4]) do result, success, backtrace, output @test !result end DocMeta.setdocmeta!(DocTest4, :DocTestSetup, :(x = 42); recursive = true, warn = false) - run_doctest([DocTest4]) do result, success, backtrace, output + run_doctest(nothing, [DocTest4]) do result, success, backtrace, output @test result end # DocTest5 - run_doctest([DocTest5]) do result, success, backtrace, output + run_doctest(nothing, [DocTest5]) do result, success, backtrace, output @test !result end DocMeta.setdocmeta!(DocTest5, :DocTestSetup, :(x = 42)) DocMeta.setdocmeta!(DocTest5.Submodule, :DocTestSetup, :(x = 4200)) - run_doctest([DocTest5]) do result, success, backtrace, output + run_doctest(nothing, [DocTest5]) do result, success, backtrace, output @test result end end diff --git a/test/manual.jl b/test/manual.jl new file mode 100644 index 00000000000..0d4b7be9d77 --- /dev/null +++ b/test/manual.jl @@ -0,0 +1,22 @@ +using Documenter +using Test + +@testset "Manual doctest" begin + @info "Doctesting Documenter manual" + @test doctest(Documenter) + + # Make sure that doctest() fails if there is a manual page with a failing doctest + @info "Doctesting Documenter manual w/ failing doctest" + tmpfile = joinpath(@__DIR__, "..", "docs", "src", "lib", "internals", "tmpfile.md") + write(tmpfile, """ + # Temporary source file w/ failing doctest + ```jldoctest + julia> 2 + 2 + 42 + ``` + """) + @test isfile(tmpfile) + @test !doctest(Documenter) + rm(tmpfile) + @test !isfile(tmpfile) +end diff --git a/test/runtests.jl b/test/runtests.jl index bd0237fcd50..eb67e349945 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -106,7 +106,4 @@ module HighlightSig end end -@testset "Manual doctest" begin - push!(ARGS, "doctest-only") - include(joinpath(@__DIR__, "..", "docs", "make.jl")) -end +include("manual.jl")