use Clang to wrap PROJ 6.1.0

Project.toml

@@ -3,6 +3,7 @@ uuid = "9a7e659c-8ee8-5706-894e-f68f43bc57ea"
 
 [deps]
 BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
+CEnum = "fa961155-64e5-5f13-b03f-caf6b980ea82"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
@@ -11,3 +12,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 [compat]
 BinaryProvider = "≥ 0.5.0"
 julia = "≥ 1.0.0"
+CEnum = "≥ 0.2.0"

gen/doc.jl

@@ -0,0 +1,150 @@
+# this file is included by wrap.jl and provides several functions for building docstrings
+
+"""Build docstring for a function from a Doxygen XML node
+
+Use argpos argument permutation as is used in the rewritten code"""
+function build_function(node::EzXML.Node, argpos::Vector{Int})
+    io = IOBuffer()
+
+    # code block with function definition
+    print(io, "    ", text(node, "name"), "(")
+    nspace = position(io)
+    params = findall("param", node)
+    if isempty(params)
+        print(io, ")")
+    else
+        params = params[argpos]
+    end
+    for param in params
+        param !== first(params) && print(io, " "^nspace) # align
+        islast = param === last(params)
+        print(io, text(param, "type"))
+        # declname not always present (with void)
+        lastchar = islast ? ")" : ",\n"
+        if !isempty(findall("declname", param))
+            print(io, " ", text(param, "declname"), lastchar)
+        else
+            print(io, lastchar)
+        end
+    end
+    println(io, " -> ", text(node, "type"))
+
+    # brief description, always 1 present, sometimes only whitespace
+    desc = strip(nodecontent(findfirst("briefdescription", node)))
+    if !isempty(desc)
+        println(io, "\n", text(node, "briefdescription"))
+    end
+
+    # parameters
+    params = findall("detaileddescription/para/parameterlist/parameteritem", node)
+    if !isempty(params)
+        params = params[argpos]
+        println(io, "\n### Parameters")
+
+        for param in params
+            print(io, "* **", text(param, "parameternamelist"), "**: ")
+            println(io, text(param, "parameterdescription"))
+        end
+    end
+
+    # returns
+    return_elems = findall("detaileddescription/para/simplesect[@kind='return']", node)
+    if !isempty(return_elems)
+        println(io, "\n### Returns")
+        println(io, text(return_elems[1], "para"))
+    end
+
+    String(take!(io))
+end
+
+"Build a one line docstring consisting of only the brief description from an XML node"
+function brief_description(node::EzXML.Node)
+    # brief description, always 1 present, sometimes only whitespace
+    strip(nodecontent(findfirst("briefdescription", node)))
+end
+
+"Compose a Markdown docstring based on a Doxygen XML element"
+function build_docstring(node::EzXML.Node, argpos::Union{Vector, Nothing})
+    kind = node["kind"]
+    if kind == "function"
+        build_function(node, argpos)
+    elseif kind in ("enum", "define", "typedef")
+        brief_description(node)
+    else
+        # "friend" and "variable" kinds remain
+        # but we leave them out, not needed
+        ""
+    end
+end
+
+"Return the text of a subelement `el` of `node`"
+function text(node::EzXML.Node, el::AbstractString)
+    s = findfirst(el, node)
+    s === nothing ? "" : strip(nodecontent(s))
+end
+
+"Wrap the single- or multi-line docstring in appropriate quotes"
+function addquotes(docstr::AbstractString)
+    if '\n' in docstr
+        string("\"\"\"\n", docstr, "\"\"\"")
+    else
+        # single line docstring
+        repr(rstrip(docstr, '.'))
+    end
+end
+
+"Get the C name out of a expression"
+function cname(ex)
+    if @capture(ex, function f_(args__) ccall((a_, b_), xs__) end)
+        String(eval(a))
+    else
+        # TODO make MacroTools.namify work for structs and macros
+        if MacroTools.isexpr(ex) && ex.head === :struct
+            String(ex.args[2])
+        elseif MacroTools.isexpr(ex) && ex.head === :macrocall
+            # if the enum has a type specified
+            String(ex.args[3].args[1])
+        else
+            String(namify(ex))
+        end
+    end
+end
+
+"Based on a name, find the best XML node to generate docs from"
+function findnode(name::String, doc::EzXML.Document)
+    # Names are not unique. We know that kind='friend' (not sure what it is)
+    # does not give good docs and is never the only one, so we skip those.
+    # First we use XPath to find all nodes with this name and not kind='friend'.
+    memberdef = "/doxygen/compounddef/sectiondef/memberdef"
+    nofriend = "not(@kind='friend')"  # :-(
+    nodes = findall("$memberdef[name='$name' and $nofriend]", doc)
+
+    if length(nodes) == 0
+        return nothing
+    elseif length(nodes) == 1
+        return first(nodes)
+    else
+        # If we get multiple nodes back, we have to select the best one.
+        # Looking at the documentation, sometimes there are two similar docstrings,
+        # but one comes from a .cpp file, as seen in location's file attribute,
+        # and the other comes from a .h file (.c is the third option).
+        # Even though this is a C binding, the .cpp node includes the argument names
+        # which makes for an easier to read docstring, since they can be referenced
+        # to the argument names in the parameters list.
+        # Therefore if .cpp is one of the options, go for that.
+
+        # ExXML uses libxml2 which only supports XPath 1.0, meaning
+        # ends-with(@file,'.cpp') is not available, but according to
+        # https://stackoverflow.com/a/11857166/2875964 we can rewrite this as
+        cpp = "'.cpp' = substring(@file, string-length(@file) - string-length('.cpp') +1)"
+
+        for node in nodes
+            cppnode = findfirst("location[$cpp]/..", node)
+            if cppnode !== nothing
+                return cppnode
+            end
+        end
+        # .cpp not present, just pick the first
+        return first(nodes)
+    end
+end

gen/wrap_proj.jl

@@ -0,0 +1,154 @@
+#=
+Run this file to regenerate `proj_c.jl` and `proj_common.jl`.
+
+It expects a PROJ install in the deps folder, run `build Proj4` in Pkg mode
+if these are not in place.
+
+The wrapped PROJ version and provided PROJ version should be kept in sync.
+So when updating the PROJBuilder provided version, also rerun this wrapper.
+This way we ensure that the provided library has the same functions available
+as the wrapped one. Furthermore this makes sure constants in `proj_common.jl`
+like `PROJ_VERSION_PATCH`, which are just literals, are correct.
+
+Several custom transformations are applied that should make using this package more convenient.
+- docstrings are added, created from PROJ Doxygen XML output
+- functions that return a Cstring are wrapped in unsafe_string to return a String
+- functions that return a Ptr{Cstring} are wrapped in unsafe_loadstringlist to return a Vector{String}
+- context arguments become keyword arguments, defaulting to C_NULL meaning default context
+
+These transformations are based on the code developed for GDAL.jl, see
+https://github.com/JuliaGeo/GDAL.jl/blob/master/gen/README.md for more information
+on how to construct the PROJ Doxygen XML file needed here.
+=#
+
+using Clang  # needs a post 0.9.1 release with #231 and #232
+using MacroTools
+using EzXML
+
+const xmlpath = joinpath(@__DIR__, "doxygen.xml")
+
+# several functions for building docstrings
+include(joinpath(@__DIR__, "doc.jl"))
+
+
+"""
+Custom rewriter for Clang.jl's C wrapper
+
+Gets called with all expressions in a header file, or all expressiong in a common file.
+If available, it adds docstrings before every expression, such that Clang.jl prints them
+on top of the expression. The expressions themselves get sent to `rewriter(::Expr)`` for
+further treatment.
+"""
+function rewriter(xs::Vector)
+    rewritten = Any[]
+    for x in xs
+        # Clang.jl inserts strings like "# Skipping MacroDefinition: X"
+        # keep these to get a sense of what we are missing
+        if x isa String
+            push!(rewritten, x)
+            continue
+        end
+        @assert x isa Expr
+
+        x2, argpos = rewriter(x)
+        name = cname(x)
+        node = findnode(name, doc)
+        docstr = node === nothing ? "" : build_docstring(node, argpos)
+        isempty(docstr) || push!(rewritten, addquotes(docstr))
+        push!(rewritten, x2)
+    end
+    rewritten
+end
+
+"Make the arg at position i a keyword and move it to the back in the argpos permutation"
+function keywordify!(fargs2, argpos, i)
+    if i === nothing
+        return nothing
+    else
+        arg = fargs2[i]
+        fargs2[i] = :($arg = C_NULL)
+        # in optpos is does not have to be at i anymore if it already was moved
+        argoptpos = findfirst(==(i), argpos)
+        splice!(argpos, argoptpos)
+        push!(argpos, i)  # add it to the end
+    end
+end
+
+"Rewrite expressions using the transformations listed at the top of this file"
+function rewriter(x::Expr)
+    if @capture(x,
+        function f_(fargs__)
+            ccall(fname_, rettype_, argtypes_, argvalues__)
+        end
+    )
+        # it is a function wrapper around a ccall
+        n = length(fargs)
+        # keep track of how we order arguments, such that we can do the same in the docs
+        argpos = collect(1:n)
+
+        fargs2 = copy(fargs)
+        if !isempty(fargs)
+            # ctx is always the first argument
+            if fargs[1] === :ctx
+                keywordify!(fargs2, argpos, 1)
+            end
+            # make all options optional
+            optpos = findfirst(==(:options), fargs)
+            keywordify!(fargs2, argpos, optpos)
+            if f === :proj_create_from_wkt
+                optpos = findfirst(==(:out_warnings), fargs)
+                keywordify!(fargs2, argpos, optpos)
+                optpos = findfirst(==(:out_grammar_errors), fargs)
+                keywordify!(fargs2, argpos, optpos)
+            end
+            # apply the argument ordering permutation
+            fargs2 = fargs2[argpos]
+        end
+
+        # bind the ccall such that we can easily wrap it
+        cc = :(ccall($fname, $rettype, $argtypes, $(argvalues...)))
+
+        cc2 = if rettype == :Cstring
+            :(aftercare($cc))
+        elseif rettype == :(Ptr{Cstring})
+            :(aftercare($cc))
+        else
+            cc
+        end
+
+        # stitch the modified function expression back together
+        x2 = :(function $f($(fargs2...))
+            $cc2
+        end) |> prettify
+        x2, argpos
+    else
+        # do not modify expressions that are no ccall function wrappers
+        # argument positions do not apply, but something still needs to be returned
+        argpos = nothing
+        x, argpos
+    end
+end
+
+# parse GDAL's Doxygen XML file
+const doc = readxml(xmlpath)
+
+# should be here if you pkg> build Proj4
+includedir = normpath(joinpath(@__DIR__, "..", "deps", "usr", "include"))
+headerfiles = [joinpath(includedir, "proj.h")]
+
+wc = init(; headers = headerfiles,
+            output_file = joinpath(@__DIR__, "..", "src", "proj_c.jl"),
+            common_file = joinpath(@__DIR__, "..", "src", "proj_common.jl"),
+            clang_includes = [includedir, CLANG_INCLUDE],
+            clang_args = ["-I", includedir],
+            header_wrapped = (root, current) -> root == current,
+            header_library = x -> "libproj",
+            clang_diagnostics = true,
+            rewriter = rewriter,
+)
+
+run(wc)
+
+# delete Clang.jl helper files
+rm(joinpath(@__DIR__, "..", "src", "LibTemplate.jl"))
+rm(joinpath(@__DIR__, "..", "src", "ctypes.jl"))

src/Proj4.jl

@@ -1,6 +1,7 @@
 module Proj4
 
 using Libdl
+using CEnum
 
 export Projection, # proj_types.jl
        transform, transform!,  # proj_functions.jl
@@ -14,14 +15,11 @@ if !isfile(depsjl_path)
 end
 include(depsjl_path)
 
-# Module initialization function
-function __init__()
-    # Always check your dependencies from `deps.jl`
-    check_deps()
-end
-
 include("projection_codes.jl") # ESRI and EPSG projection strings
 include("proj_capi.jl") # low-level C-facing functions (corresponding to src/proj_api.h)
+include("proj_common.jl")
+include("proj_c.jl")
+include("error.jl")
 
 function _version()
     m = match(r"(\d+).(\d+).(\d+),.+", _get_release())
@@ -45,4 +43,38 @@ include("proj_functions.jl") # user-facing proj functions
 "Get a global error string in human readable form"
 error_message() = _strerrno()
 
+"""
+Load a null-terminated list of strings
+
+It takes a `PROJ_STRING_LIST`, which is a `Ptr{Cstring}`, and returns a `Vector{String}`.
+"""
+function unsafe_loadstringlist(ptr::Ptr{Cstring})
+    strings = Vector{String}()
+    (ptr == C_NULL) && return strings
+    i = 1
+    cstring = unsafe_load(ptr, i)
+    while cstring != C_NULL
+        push!(strings, unsafe_string(cstring))
+        i += 1
+        cstring = unsafe_load(ptr, i)
+    end
+    strings
+end
+
+const PROJ_LIB = Ref{String}()
+
+"Module initialization function"
+function __init__()
+    # Always check your dependencies from `deps.jl`
+    check_deps()
+
+    # register custom error handler
+    funcptr = @cfunction(log_func, Ptr{Cvoid}, (Ptr{Cvoid}, Cint, Cstring))
+    proj_log_func(C_NULL, funcptr)
+
+    # point to the location of the provided shared resources
+    PROJ_LIB[] = abspath(@__DIR__, "..", "deps", "usr", "share", "proj")
+    proj_context_set_search_paths(1, [PROJ_LIB[]])
+end
+
 end # module