move the TOML parser to Base and implement code loading on top of the…

… Base TOML parser (#36018)

base/Base.jl

@@ -340,6 +340,8 @@ include("threadcall.jl")
 
 # code loading
 include("uuid.jl")
+include("pkgid.jl")
+include("toml_parser.jl")
 include("loading.jl")
 
 # misc useful functions & macros

base/initdefs.jl

@@ -234,7 +234,7 @@ function load_path_expand(env::AbstractString)::Union{String, Nothing}
         # if you put a `@` in LOAD_PATH manually, it's expanded late
         env == "@" && return active_project(false)
         env == "@." && return current_project()
-        env == "@stdlib" && return Sys.STDLIB
+        env == "@stdlib" && return Sys.STDLIB::String
         env = replace(env, '#' => VERSION.major, count=1)
         env = replace(env, '#' => VERSION.minor, count=1)
         env = replace(env, '#' => VERSION.patch, count=1)

base/pkgid.jl

@@ -0,0 +1,43 @@
+struct PkgId
+    uuid::Union{UUID,Nothing}
+    name::String
+
+    PkgId(u::UUID, name::AbstractString) = new(UInt128(u) == 0 ? nothing : u, name)
+    PkgId(::Nothing, name::AbstractString) = new(nothing, name)
+end
+PkgId(name::AbstractString) = PkgId(nothing, name)
+
+function PkgId(m::Module, name::String = String(nameof(moduleroot(m))))
+    uuid = UUID(ccall(:jl_module_uuid, NTuple{2, UInt64}, (Any,), m))
+    UInt128(uuid) == 0 ? PkgId(name) : PkgId(uuid, name)
+end
+
+==(a::PkgId, b::PkgId) = a.uuid == b.uuid && a.name == b.name
+
+function hash(pkg::PkgId, h::UInt)
+    h += 0xc9f248583a0ca36c % UInt
+    h = hash(pkg.uuid, h)
+    h = hash(pkg.name, h)
+    return h
+end
+
+show(io::IO, pkg::PkgId) =
+    print(io, pkg.name, " [", pkg.uuid === nothing ? "top-level" : pkg.uuid, "]")
+
+function binpack(pkg::PkgId)
+    io = IOBuffer()
+    write(io, UInt8(0))
+    uuid = pkg.uuid
+    write(io, uuid === nothing ? UInt128(0) : UInt128(uuid))
+    write(io, pkg.name)
+    return String(take!(io))
+end
+
+function binunpack(s::String)
+    io = IOBuffer(s)
+    @assert read(io, UInt8) === 0x00
+    uuid = read(io, UInt128)
+    name = read(io, String)
+    return PkgId(UUID(uuid), name)
+end
+

stdlib/TOML/src/parser.jl → base/toml_parser.jl

@@ -1,3 +1,5 @@
+module TOML
+
 using Base: IdSet
 
 # In case we do not have the Dates stdlib available
@@ -96,7 +98,7 @@ function Parser(str::String; filepath=nothing)
             String[],             # dotted_keys
             UnitRange{Int}[],     # chunks
             IdSet{TOMLDict}(),    # inline_tables
-            IdSet{Any}(),   # static_arrays
+            IdSet{Any}(),         # static_arrays
             IdSet{TOMLDict}(),    # defined_tables
             root,
             filepath,
@@ -1164,3 +1166,5 @@ function take_chunks(l::Parser, unescape::Bool)::String
     empty!(l.chunks)
     return unescape ? unescape_string(str) : str
 end
+
+end

base/uuid.jl

@@ -8,6 +8,7 @@
 struct UUID
     value::UInt128
 end
+UUID(u::UUID) = u
 UUID(u::NTuple{2, UInt64}) = UUID((UInt128(u[1]) << 64) | UInt128(u[2]))
 UUID(u::NTuple{4, UInt32}) = UUID((UInt128(u[1]) << 96) | (UInt128(u[2]) << 64) |
                                   (UInt128(u[3]) << 32) | UInt128(u[4]))

stdlib/Profile/src/Profile.jl

@@ -271,7 +271,7 @@ function short_path(spath::Symbol, filenamecache::Dict{Symbol, String})
                     for proj in Base.project_names
                         project_file = joinpath(root, proj)
                         if Base.isfile_casesensitive(project_file)
-                            pkgid = Base.project_file_name_uuid(project_file, "")
+                            pkgid = Base.project_file_name_uuid(project_file, "", Base.TOMLCache())
                             isempty(pkgid.name) && return path # bad Project file
                             # return the joined the module name prefix and path suffix
                             path = path[nextind(path, sizeof(root)):end]

stdlib/REPL/src/REPLCompletions.jl

@@ -585,23 +585,15 @@ end
 
 function project_deps_get_completion_candidates(pkgstarts::String, project_file::String)
     loading_candidates = String[]
-    open(project_file) do io
-        state = :top
-        for line in eachline(io)
-            if occursin(Base.re_section, line)
-                state = occursin(Base.re_section_deps, line) ? :deps : :other
-            elseif state === :top
-                if (m = match(Base.re_name_to_string, line)) !== nothing
-                    root_name = String(m.captures[1])
-                    startswith(root_name, pkgstarts) && push!(loading_candidates, root_name)
-                end
-            elseif state === :deps
-                if (m = match(Base.re_key_to_string, line)) !== nothing
-                    dep_name = m.captures[1]
-                    startswith(dep_name, pkgstarts) && push!(loading_candidates, dep_name)
-                end
-            end
-        end
+    p = Base.TOML.Parser()
+    Base.TOML.reinit!(p, read(project_file, String); filepath=project_file)
+    d = Base.TOML.parse(p)
+    pkg = get(d, "name", nothing)
+    if pkg !== nothing && startswith(pkg, pkgstarts)
+        push!(loading_candidates, pkg)
+    end
+    for (pkg, _) in get(d, "deps", [])
+        startswith(pkg, pkgstarts) && push!(loading_candidates, pkg)
     end
     return Completion[PackageCompletion(name) for name in loading_candidates]
 end

stdlib/TOML/src/TOML.jl

@@ -1,7 +1,12 @@
 module TOML
 
 module Internals
-    include("parser.jl")
+    # The parser is defined in Base
+    using Base.TOML: Parser, parse, tryparse, ParserError, isvalid_barekey_char, reinit!
+    # Put the error instances in this module
+    for errtype in instances(Base.TOML.ErrorType)
+        @eval using Base.TOML: $(Symbol(errtype))
+    end
     # We put the printing functionality in a separate module since It
     # defines a function `print` and we don't want that to collide with normal
     # usage of `(Base.)print` in other files

test/loading.jl

@@ -166,9 +166,10 @@ end
                 d = findfirst(line -> line == "[deps]", p)
                 t = findfirst(line -> startswith(line, "This"), p)
                 # look up various packages by name
-                root = Base.explicit_project_deps_get(project_file, "Root")
-                this = Base.explicit_project_deps_get(project_file, "This")
-                that = Base.explicit_project_deps_get(project_file, "That")
+                cache = Base.TOMLCache()
+                root = Base.explicit_project_deps_get(project_file, "Root", cache)
+                this = Base.explicit_project_deps_get(project_file, "This", cache)
+                that = Base.explicit_project_deps_get(project_file, "That", cache)
                 # test that the correct answers are given
                 @test root == (something(n, N+1) ≥ something(d, N+1) ? nothing :
                                something(u, N+1) < something(d, N+1) ? root_uuid : proj_uuid)
@@ -191,6 +192,23 @@ Base.ACTIVE_PROJECT[] = nothing
 
 @test load_path() == [abspath("project","Project.toml")]
 
+
+# locate `tail(names)` package by following the search path graph through `names` starting from `where`
+function recurse_package(where::PkgId, name::String, names::String...)
+    pkg = identify_package(where, name, Base.TOMLCache())
+    pkg === nothing && return nothing
+    return recurse_package(pkg, names...)
+end
+
+recurse_package(pkg::String) = identify_package(pkg)
+recurse_package(where::PkgId, pkg::String) = identify_package(where, pkg, Base.TOMLCache())
+
+function recurse_package(name::String, names::String...)
+    pkg = identify_package(name)
+    pkg === nothing && return nothing
+    return recurse_package(pkg, names...)
+end
+
 @testset "project & manifest identify_package & locate_package" begin
     local path
     for (names, uuid, path) in [
@@ -201,14 +219,14 @@ Base.ACTIVE_PROJECT[] = nothing
         ("Foo.Qux", "b5ec9b9c-e354-47fd-b367-a348bdc8f909", "project/deps/Qux.jl"                ),
     ]
         n = map(String, split(names, '.'))
-        pkg = identify_package(n...)
+        pkg = recurse_package(n...)
         @test pkg == PkgId(UUID(uuid), n[end])
         @test joinpath(@__DIR__, normpath(path)) == locate_package(pkg)
     end
     @test identify_package("Baz") == nothing
     @test identify_package("Qux") == nothing
     @testset "equivalent package names" begin
-        local classes = [
+         classes = [
             ["Foo"],
             ["Bar", "Foo.Bar"],
             ["Foo.Baz", "Bar.Baz", "Foo.Bar.Baz"],
@@ -221,15 +239,15 @@ Base.ACTIVE_PROJECT[] = nothing
         for i = 1:length(classes)
             A = classes[i]
             for x in A
-                X = identify_package(map(String, split(x, '.'))...)
+                X = recurse_package(map(String, split(x, '.'))...)
                 for y in A
-                    Y = identify_package(map(String, split(y, '.'))...)
+                    Y = recurse_package(map(String, split(y, '.'))...)
                     @test X == Y
                 end
                 for j = i+1:length(classes)
                     B = classes[j]
                     for z in B
-                        Z = identify_package(map(String, split(z, '.'))...)
+                        Z = recurse_package(map(String, split(z, '.'))...)
                         @test X != Z
                     end
                 end
@@ -384,8 +402,6 @@ const envs = Dict{String,Any}()
 append!(empty!(DEPOT_PATH), depots)
 
 @testset "load code uniqueness" begin
-    @show UUIDS
-    @show depots
     @test allunique(UUIDS)
     @test allunique(depots)
     @test allunique(DEPOT_PATH)

test/precompile.jl

@@ -244,7 +244,7 @@ try
     # use _require_from_serialized to ensure that the test fails if
     # the module doesn't reload from the image:
     @test_logs (:warn, "Replacing module `$Foo_module`") begin
-        ms = Base._require_from_serialized(cachefile)
+        ms = Base._require_from_serialized(cachefile, Base.TOMLCache())
         @test isa(ms, Array{Any,1})
     end