Update doctest API

doctest() can now also doctest manual pages which makes it easier to
include doctesting as part of the test suite.

docs/src/man/doctests.md

@@ -311,67 +311,46 @@ julia> @time [1,2,3,4]
     keyword argument are all applied to each doctest.
 
 
-## Doctesting Without Building the Docs
+## Doctesting as Part of Testing
 
-Documenter has a few ways to verify the doctests without having to run a potentially
-expensive full documentation build.
+Documenter provides the [`doctest`](@ref) function which can be used to verify all doctests
+independently of manual builds. It simply returns `true` if the doctests pass, or `false`
+if they do not.
 
-### Doctesting docstrings only
-
-An option for doctesting just the docstrings of a particular module (and all its submodules)
-is to use the [`doctest`](@ref) function. It takes a list of modules as an argument and runs
-all the doctests in all the docstrings it finds. This can be handy for quick tests when
-writing docstrings of a package.
-
-[`doctest`](@ref) will return `true` or `false`, depending on whether the doctests pass or
-not, making it easy to include a doctest of all the docstrings in the package test suite:
-
-```julia
-using MyPackage, Documenter, Test
-@test doctest([MyPackage])
-```
-
-Note that you still need to make sure that all the necessary [Module-level metadata](@ref)
-for the doctests is set up before [`doctest`](@ref) is called.
-
-### Doctesting without a full build
-
-An alternative, which also runs doctests on the manual pages, but still skips most other
-build steps, is to pass `doctest = :only` to [`makedocs`](@ref).
-
-This also makes it more practical to include doctests as part of the normal test suite of a
-package. One option to set it up is to make the `doctest` keyword depend on command line
-arguments passed to the `make.jl` script:
+For example, it can be used to verify doctests as part of the normal test suite by having
+e.g. the following in `runtests.jl`:
 
 ```julia
-makedocs(...,
-    doctest = ("doctest-only" in ARGS) ? :only : true
-)
+using Test, Documenter, MyPackage
+@testset "Doctesting" begin
+    @test doctest(MyPackage)
+end
 ```
 
-Now, the `make.jl` script can be run on the command line as `julia docs/make.jl
-doctest-only` and it will only run the doctests. On doctest failure, the `makedocs` throws
-an error and `julia` exits with a non-zero exit code.
+By default, it will also attempt to verify all the doctests on manual `.md` files, which it
+assumes are located under `docs/src`. This can be configured or disabled with the `manual`
+keyword (see [`doctest`](@ref) for more information).
 
-For running the doctests as part of the standard test suite, the  `docs/make.jl` can simply
-be `include`d in the `test/runtest.jl` file:
+For example, to test a package that does have separate manual pages, just docstrings,
+`runtests.jl` can be set up as follows:
 
 ```julia
-push!(ARGS, "doctest-only")
-include(joinpath(@__DIR__, "..", "docs", "make.jl"))
+using Test, Documenter, MyPackage
+@testset "Doctesting" begin
+    @test doctest(MyPackage; manual = false)
+end
 ```
 
-The `push!` to `ARGS` emulates the passing of the `doctest-only` command line argument.
-
-Note that, for this to work, you need to add Documenter and all the other packages that get
-loaded in `make.jl`, or in the doctest, as test dependencies.
+Note that you still need to make sure that all the necessary [Module-level metadata](@ref)
+for the doctests is set up before [`doctest`](@ref) is called. Also, you need to add
+Documenter and all the other packages you are loading in the doctests as test dependencies.
 
 
 ## Fixing Outdated Doctests
 
-To fix outdated doctests, the `doctest` flag to [`makedocs`](@ref) can be set to
-`doctest = :fix`. This will run the doctests, and overwrite the old results with
-the new output.
+To fix outdated doctests, the [`doctest`](@ref) function can be called with `fix = true`,
+the `doctest = :fix` keyword can be passed to [`makedocs`](@ref). This will run the
+doctests, and overwrite the old results with the new output.
 
 !!! note
 

docs/src/showcase.md

@@ -0,0 +1,33 @@
+# Hidden showcase page
+
+Currently exists just so that there would be doctests to run in manual pages of Documenter's
+manual. This page does not show up in navigation.
+
+```jldoctest
+julia> 2 + 2
+4
+```
+
+The following doctests needs doctestsetup:
+
+```jldoctest; setup=:(using Documenter)
+julia> Documenter.Utilities.splitexpr(:(Foo.Bar.baz))
+(:(Foo.Bar), :(:baz))
+```
+
+Let's also try `@meta` blocks:
+
+```@meta
+DocTestSetup = quote
+  f(x) = x^2
+end
+```
+
+```jldoctest
+julia> f(2)
+4
+```
+
+```@meta
+DocTestSetup = nothing
+```

src/Documenter.jl

@@ -826,20 +826,71 @@ function getenv(regex::Regex)
 end
 
 """
-    doctest(modules::AbstractVector{Module}) -> Bool
+    doctest(package::Module; kwargs...) -> Bool
+
+Convenience method that runs and checks all the doctests for a given Julia package.
+`package` must be the `Module` object corresponding to the top-level module of the package.
+Returns `true` if the doctesting was successful and false if any error occurred.
+
+# Keywords
+
+**`manual`** controls how manual pages are handled. By default (`manual = true`), `doctest`
+assumes that manual pages are located under `docs/src`. If that is not the case, the
+`manual` keyword argument can be passed to specify the directory. Setting `manual = false`
+will skip doctesting of manual pages altogether.
+
+Additional keywords are passed on to the main [`doctest`](@ref) method.
+"""
+function doctest(package::Module; manual=true, kwargs...)
+    if pathof(package) === nothing
+        throw(ArgumentError("$(package) is not a top-level package module."))
+    end
+    source = nothing
+    if manual === true
+         source = normpath(joinpath(dirname(pathof(package)), "..", "docs", "src"))
+         isdir(source) || throw(ArgumentError("""
+         Package $(package) does not have a documentation source directory at standard location.
+         Searched at: $(source)
+         If ...
+         """))
+    end
+    doctest(source, [package]; kwargs...)
+end
 
-Runs all the doctests in the given modules. Returns `true` if the doctesting was successful
-and false if any error occurred.
 """
-function doctest(modules::AbstractVector{Module})
+    doctest(source, modules; kwargs...) -> Bool
+
+Runs all the doctests in the given modules and on manual pages under the `source` directory.
+Returns `true` if the doctesting was successful and `false` if any error occurred.
+
+The manual pages are searched recursively in subdirectories of `source` too. Doctesting of
+manual pages can be disabled if `source` is set to `nothing`.
+
+# Keywords
+
+**`fix`**, if set to `true`, updates all the doctests that fail with the correct output
+(default `false`).
+
+!!! warning
+    When running `doctest(...; fix=true)`, Documenter will modify the Markdown and Julia
+    source files. It is strongly recommended that you only run it on packages in Pkg's
+    develop mode and commit any staged changes. You should also review all the changes made
+    by `doctest` before committing them, as there may be edge cases when the automatic
+    fixing fails.
+"""
+function doctest(source::Union{AbstractString,Nothing}, modules::AbstractVector{Module}; fix=false)
     dir = mktempdir()
     @debug "Doctesting in temporary directory: $(dir)" modules
-    mkdir(joinpath(dir, "src"))
+    if source === nothing
+        source = joinpath(dir, "src")
+        mkdir(source)
+    end
     r = try
         makedocs(
             root = dir,
+            source = source,
             sitename = "",
-            doctest = :only,
+            doctest = fix ? :fix : :only,
             modules = modules,
         )
         true

test/doctests/doctestapi.jl

@@ -96,44 +96,44 @@ end
 
 @testset "Documenter.doctest" begin
     # DocTest1
-    run_doctest([DocTest1]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest1]) do result, success, backtrace, output
         @test result
     end
 
     # DocTest2
-    run_doctest([DocTest2]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest2]) do result, success, backtrace, output
         @test !result
     end
 
     # DocTest3
-    run_doctest([DocTest3]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest3]) do result, success, backtrace, output
         @test !result
     end
     DocMeta.setdocmeta!(DocTest3, :DocTestSetup, :(x = 42))
-    run_doctest([DocTest3]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest3]) do result, success, backtrace, output
         @test result
     end
 
     # DocTest4
-    run_doctest([DocTest4]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest4]) do result, success, backtrace, output
         @test !result
     end
     DocMeta.setdocmeta!(DocTest4, :DocTestSetup, :(x = 42))
-    run_doctest([DocTest4]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest4]) do result, success, backtrace, output
         @test !result
     end
     DocMeta.setdocmeta!(DocTest4, :DocTestSetup, :(x = 42); recursive = true, warn = false)
-    run_doctest([DocTest4]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest4]) do result, success, backtrace, output
         @test result
     end
 
     # DocTest5
-    run_doctest([DocTest5]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest5]) do result, success, backtrace, output
         @test !result
     end
     DocMeta.setdocmeta!(DocTest5, :DocTestSetup, :(x = 42))
     DocMeta.setdocmeta!(DocTest5.Submodule, :DocTestSetup, :(x = 4200))
-    run_doctest([DocTest5]) do result, success, backtrace, output
+    run_doctest(nothing, [DocTest5]) do result, success, backtrace, output
         @test result
     end
 end

test/manual.jl

@@ -0,0 +1,22 @@
+using Documenter
+using Test
+
+@testset "Manual doctest" begin
+    @info "Doctesting Documenter manual"
+    @test doctest(Documenter)
+
+    # Make sure that doctest() fails if there is a manual page with a failing doctest
+    @info "Doctesting Documenter manual w/ failing doctest"
+    tmpfile = joinpath(@__DIR__, "..", "docs", "src", "lib", "internals", "tmpfile.md")
+    write(tmpfile, """
+    # Temporary source file w/ failing doctest
+    ```jldoctest
+    julia> 2 + 2
+    42
+    ```
+    """)
+    @test isfile(tmpfile)
+    @test !doctest(Documenter)
+    rm(tmpfile)
+    @test !isfile(tmpfile)
+end

test/runtests.jl

@@ -106,7 +106,4 @@ module HighlightSig
     end
 end
 
-@testset "Manual doctest" begin
-    push!(ARGS, "doctest-only")
-    include(joinpath(@__DIR__, "..", "docs", "make.jl"))
-end
+include("manual.jl")