From 2787ef446253d8ff66fbd983a6a565bcd8b8f2f8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 27 Aug 2020 18:32:19 +0200 Subject: [PATCH] WIP: use the stdlib version of TOML (#1984) * use the stdlib version of TOML * stop TOML testing --- .travis.yml | 14 - Project.toml | 1 + ext/TOML/.gitignore | 3 - ext/TOML/.travis.yml | 15 - ext/TOML/LICENSE.md | 22 - ext/TOML/Project.toml | 12 - ext/TOML/README.md | 37 - ext/TOML/REQUIRE | 1 - ext/TOML/appveyor.yml | 33 - ext/TOML/src/TOML.jl | 52 - ext/TOML/src/parser.jl | 890 ------------------ ext/TOML/src/print.jl | 112 --- ext/TOML/test/runtests.jl | 690 -------------- src/API.jl | 2 +- src/Artifacts.jl | 2 +- src/Operations.jl | 8 +- src/Pkg.jl | 6 +- src/Types.jl | 20 +- src/manifest.jl | 41 +- src/project.jl | 32 +- test/artifacts.jl | 12 +- test/pkg.jl | 3 +- test/repl.jl | 5 +- .../BuildProjectFixedDeps/Manifest.toml | 9 +- .../BuildProjectFixedDeps/Project.toml | 1 + .../BuildProjectFixedDeps/deps/build.jl | 2 +- test/utils.jl | 5 +- 27 files changed, 76 insertions(+), 1954 deletions(-) delete mode 100644 ext/TOML/.gitignore delete mode 100644 ext/TOML/.travis.yml delete mode 100644 ext/TOML/LICENSE.md delete mode 100644 ext/TOML/Project.toml delete mode 100644 ext/TOML/README.md delete mode 100644 ext/TOML/REQUIRE delete mode 100644 ext/TOML/appveyor.yml delete mode 100644 ext/TOML/src/TOML.jl delete mode 100644 ext/TOML/src/parser.jl delete mode 100644 ext/TOML/src/print.jl delete mode 100644 ext/TOML/test/runtests.jl diff --git a/.travis.yml b/.travis.yml index 0e3cad06e4..2eae04b8d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,20 +44,6 @@ jobs: - julia --project -e 'using Pkg; Pkg.activate("docs"); Pkg.instantiate(); Pkg.develop(PackageSpec(path = pwd()))' - julia --project=docs --color=yes docs/make.jl pdf after_success: skip - - name: TOML (Julia 1.3) - stage: test - os: linux - julia: 1.3 - script: - - julia --project=ext/TOML -e 'using Pkg; Pkg.test()' - after_success: skip - - name: TOML (Julia nightly) - stage: test - os: linux - julia: nightly - script: - - julia --project=ext/TOML -e 'using Pkg; Pkg.test()' - after_success: skip after_success: - julia --project=test/coverage -e 'using Pkg; Pkg.instantiate(); diff --git a/Project.toml b/Project.toml index bbf1ec9e62..5a3a5065d1 100644 --- a/Project.toml +++ b/Project.toml @@ -15,6 +15,7 @@ Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [extras] diff --git a/ext/TOML/.gitignore b/ext/TOML/.gitignore deleted file mode 100644 index 8c960ec808..0000000000 --- a/ext/TOML/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.jl.cov -*.jl.*.cov -*.jl.mem diff --git a/ext/TOML/.travis.yml b/ext/TOML/.travis.yml deleted file mode 100644 index 6902f1f2db..0000000000 --- a/ext/TOML/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: julia -os: - - linux - - osx -julia: - - 0.6 - - nightly -notifications: - email: false -script: - - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - - julia -e 'Pkg.clone(pwd()); Pkg.test("TOML"; coverage=true)' -after_success: - # push coverage results to Coveralls - - julia -e 'cd(Pkg.dir("TOML")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' diff --git a/ext/TOML/LICENSE.md b/ext/TOML/LICENSE.md deleted file mode 100644 index 0f14095807..0000000000 --- a/ext/TOML/LICENSE.md +++ /dev/null @@ -1,22 +0,0 @@ -The TOML.jl package is licensed under the MIT "Expat" License: - -> Copyright (c) 2016: Art Wild. -> -> Permission is hereby granted, free of charge, to any person obtaining a copy -> of this software and associated documentation files (the "Software"), to deal -> in the Software without restriction, including without limitation the rights -> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -> copies of the Software, and to permit persons to whom the Software is -> furnished to do so, subject to the following conditions: -> -> The above copyright notice and this permission notice shall be included in all -> copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -> SOFTWARE. -> diff --git a/ext/TOML/Project.toml b/ext/TOML/Project.toml deleted file mode 100644 index c1a6c652ce..0000000000 --- a/ext/TOML/Project.toml +++ /dev/null @@ -1,12 +0,0 @@ -name = "TOML" -uuid = "037cace4-c66a-5006-a6b7-c26ba1b2f83e" -desc = "The TOML parser for Julia's package manager" - -[deps] -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[targets] -test = ["Test"] diff --git a/ext/TOML/README.md b/ext/TOML/README.md deleted file mode 100644 index 7c617c8338..0000000000 --- a/ext/TOML/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# TOML.jl - -A [TOML v0.4.0](https://github.com/toml-lang/toml) parser for Julia. - -[![Build Status](https://travis-ci.org/wildart/TOML.jl.svg?branch=master)](https://travis-ci.org/wildart/TOML.jl) -[![Coverage Status](https://coveralls.io/repos/wildart/TOML.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/wildart/TOML.jl?branch=master) -[![Build status](https://ci.appveyor.com/api/projects/status/quhhe2m3e9vbim6u?svg=true)](https://ci.appveyor.com/project/wildart/toml-jl) - -**Installation**: `julia> Pkg.clone("https://github.com/wildart/TOML.jl.git")` - -## Basic Usage - -```julia - -julia> using TOML - -julia> TOML.parse(""" - name = "value" - """) -Dict{String,Any} with 1 entry: - "name" => "value" - -julia> TOML.parsefile("etc/example.toml") -``` - -## Documentation -```julia -TOML.print(io::IO, a::AbstractDict) -``` -Writes a TOML representation to the supplied `IO`. - -```julia -TOML.parse(s::AbstractString) -TOML.parse(io::IO) -TOML.parsefile(filename::AbstractString) -``` -Parses a TOML `AbstractString` or `IO` stream into a nested `Array`or `Dict`. diff --git a/ext/TOML/REQUIRE b/ext/TOML/REQUIRE deleted file mode 100644 index 137767a42a..0000000000 --- a/ext/TOML/REQUIRE +++ /dev/null @@ -1 +0,0 @@ -julia 0.6 diff --git a/ext/TOML/appveyor.yml b/ext/TOML/appveyor.yml deleted file mode 100644 index 88fd4e37fe..0000000000 --- a/ext/TOML/appveyor.yml +++ /dev/null @@ -1,33 +0,0 @@ -environment: - matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" - -branches: - only: - - master - -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - -install: -# Download most recent Julia Windows binary - - ps: (new-object net.webclient).DownloadFile( - $env:JULIA_URL, - "C:\projects\julia-binary.exe") -# Run installer silently, output to C:\projects\julia - - C:\projects\julia-binary.exe /S /D=C:\projects\julia - -build_script: -# Need to convert from shallow to complete for Pkg.clone to work - - IF EXIST .git\shallow (git fetch --unshallow) - - C:\projects\julia\bin\julia -e "versioninfo(); - Pkg.clone(pwd(), \"TOML\") - -test_script: - - C:\projects\julia\bin\julia -e "Pkg.test(\"TOML\")" diff --git a/ext/TOML/src/TOML.jl b/ext/TOML/src/TOML.jl deleted file mode 100644 index f6bfee93d9..0000000000 --- a/ext/TOML/src/TOML.jl +++ /dev/null @@ -1,52 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -# TODO -# add overall how-to docstring -# create test cases of malformed TOML to see if caught and errors reported correctly - -module TOML - using Dates - - include("parser.jl") - include("print.jl") - - "Convert `TOML.Table` to `Dict{String,Any}`" - function table2dict(tbl::Union{Nothing,Table}) - tbl === nothing && return Dict{String,Any}() - return table2dict(get(tbl)) - end - - function table2dict(tbl::Table) - ret = Dict{String,Any}() - for (k,v) in tbl.values - if isa(v, Table) - ret[k] = table2dict(v) - elseif isa(v, Array) && length(v)>0 && isa(v[1], Table) - ret[k] = [table2dict(e) for e in v] - else - ret[k] = v - end - end - return ret - end - - "Parse IO input and return result as dictionary." - function parse(io::IO) - parser = Parser(io) - res = parse(parser) - length(parser.errors)>0 && throw(CompositeException(parser.errors)) - return table2dict(res) - end - - "Parse string" - function parse(str::AbstractString) - io = IOBuffer(str) - res = parse(io) - close(io) - return res - end - - "Parse file" - parsefile(filename::AbstractString) = parse(IOBuffer(read(filename))) - -end diff --git a/ext/TOML/src/parser.jl b/ext/TOML/src/parser.jl deleted file mode 100644 index 095687271a..0000000000 --- a/ext/TOML/src/parser.jl +++ /dev/null @@ -1,890 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -const DictType = Dict{String,Any} - -"TOML Table" -struct Table - values::DictType - defined::Bool -end - -Table(defined::Bool) = Table(DictType(), defined) -function Base.show(io::IO, tbl::Table, level::Int=1) - Base.print(io, "T($(tbl.defined)){\n") - for (k,v) in tbl.values - Base.print(io, "\t"^level, k, " => ") - if isa(v, Table) - Base.show(io, v, level+1) - Base.print(io, "\n") - elseif isa(v, Vector{Table}) - Base.print(io, "[\n") - for i in 1:length(v) - Base.print(io, "\t"^level, " ") - Base.show(io, v[i], level+1) - i != length(v) && Base.print(io, ",\n") - end - Base.print("\t"^level, "]\n") - else - Base.print(io, " $v\n") - end - end - Base.print(io, "\t"^(level-1), "}\n") -end -Base.getindex(tbl::Table, key::AbstractString) = tbl.values[key] -Base.haskey(tbl::Table, key::AbstractString) = haskey(tbl.values ,key) - -"Parser error exception" -struct ParserError <: Exception - lo::Int - hi::Int - msg::String -end - -"TOML Parser: Data structure for parsing process." -mutable struct Parser{IO_T <: IO} - input::IO_T - errors::Vector{ParserError} - charbuffer::Base.GenericIOBuffer{Array{UInt8,1}} - currentchar::Char - - Parser(input::IO_T) where {IO_T <: IO} = new{IO_T}(input, ParserError[], IOBuffer(), ' ') -end -Parser(input::String) = Parser(IOBuffer(input)) -pusherror!(p::Parser, l, h, msg) = push!(p.errors, ParserError(l, h, msg)) -Base.eof(p::Parser) = eof(p.input) -Base.position(p::Parser) = Int(position(p.input))+1 -Base.write(p::Parser, x) = write(p.charbuffer, x) -Base.read(p::Parser) = p.currentchar = read(p.input, Char) - -"Rewind parser input on `n` characters." -function rewind(p::Parser, n=1) - pos = position(p.input) - pos == 0 && return 0 - skip(p.input, -n) - return Int(position(p.input)) -end - -"Converts an offset to a line and a column in the original source." -function linecol(p::Parser, offset::Int) - pos = position(p) - seekstart(p.input) - line = 0 - cur = 1 - for (i,l) in enumerate(eachline(p.input, keep=true)) - if cur + length(l) > offset - return (i, offset - cur + 1) - end - cur += length(l) - line+=1 - end - seek(p.input, pos) - return (line, 0) -end - -"Determine next position in parser input" -function nextpos(p::Parser) - pos = position(p) - return pos + 1 -end - -"Peeks ahead `n` characters" -function peek(p::Parser) #, i::Int=0 - eof(p) && return nothing - res = Base.peek(p.input) - res == -1 && return nothing - return Char(res) -end - -"Returns `true` and consumes the next character if it matches `ch`, otherwise do nothing and return `false`" -function consume(p::Parser, ch::AbstractChar) - eof(p) && return false - c = peek(p) - if c == ch - read(p) - return true - else - return false - end -end - -function expect(p::Parser, ch::AbstractChar) - consume(p, ch) && return true - lo = position(p) - if eof(p) - pusherror!(p, lo, lo, "expected `$ch`, but found EOF") - else - v = peek(p) - mark(p.input) - c = read(p) - pusherror!(p, lo, lo+1, "expected `$ch`, but found `$c`") - reset(p.input) - end - return false -end - -"Consumes whitespace ('\t' and ' ') until another character (or EOF) is reached. Returns `true` if any whitespace was consumed" -function whitespace(p::Parser) - ret = false - while !eof(p) - c = read(p) - if c == '\t' || c == ' ' - ret = true - else - rewind(p) - break - end - end - return ret -end - -"Consumes the rest of the line after a comment character" -function comment(p::Parser) - !consume(p, '#') && return false - while !eof(p) - ch = read(p) - ch == '\n' && break - end - return true -end - -"Consumes a newline if one is next" -function newline(p::Parser) - if !eof(p) - n = 1 - ch = read(p.input, UInt8) - ch == 0x0a && return true - if ch == 0x0d - if !eof(p) - nch = read(p.input, UInt8) - nch == 0x0a && return true - n+=1 - end - end - rewind(p, n) - end - return false -end - -"Consume ignored symbols" -function ignore(p::Parser) - while true - whitespace(p) - !newline(p) && !comment(p) && break - end -end - -"Parse a single key name starting at next position" -function keyname(p::Parser) - s = nextpos(p) - key = if consume(p, '"') - basicstring(p, s, false) - elseif consume(p, '\'') - literalstring(p, s, false) - else - while !eof(p) - ch = read(p) - if 'a' <= ch <= 'z' || - 'A' <= ch <= 'Z' || - isdigit(ch) || - ch == '_' || - ch == '-' - write(p, ch) - else - rewind(p) - break - end - end - String(take!(p.charbuffer)) - end - - if key !== nothing && isempty(key) - pusherror!(p, s, s, "expected a key but found an empty string") - key = nothing - end - return key -end - -"Parse a path into a vector of paths" -function lookup(p::Parser) - ks = String[] - eof(p) && return ks - while true - whitespace(p) - s = keyname(p) - if s !== nothing - push!(ks, s) - else - s = integer(p, 0) - if s !== nothing - push!(ks, s) - else - return nothing - end - end - whitespace(p) - !expect(p, '.') && return ks - end -end - -"Parses a key-value separator" -function separator(p::Parser) - whitespace(p) - !expect(p, '=') && return false - whitespace(p) - return true -end - -"Insert key-value pair to table, if duplicate key found reports error" -function insertpair(p::Parser, tbl::Table, k, v, st) - if haskey(tbl, k) - pusherror!(p, st, st+length(k), "duplicate key `$k`") - else - tbl.values[k] = v - end -end - -"Parses integer with leading zeros and sign" -function integer(p::Parser, st::Int, allow_leading_zeros::Bool=false, allow_sign::Bool=false) - if allow_sign - if consume(p, '-') - write(p, '-') - elseif consume(p, '+') - write(p, '+') - end - end - nch = peek(p) - if nch === nothing - pos = nextpos(p) - pusherror!(p, pos, pos, "expected start of a numeric literal") - return nothing - else - ch = nch - if isdigit(ch) - c = read(p) - if c == '0' && !allow_leading_zeros - write(p, '0') - nch = peek(p) - if nch !== nothing - ch = nch - if isdigit(ch) - pusherror!(p, st, position(p), "leading zeroes are not allowed") - return nothing - end - end - elseif isdigit(c) - write(p, c) - end - else - # non-digit - pusherror!(p, st, position(p), "expected a digit, found `$ch`") - return nothing - end - end - - underscore = false - while !eof(p) - ch = read(p) - if isdigit(ch) - write(p, ch) - underscore = false - elseif ch == '_' && !underscore - underscore = true - else - rewind(p) - break - end - end - if underscore - pos = nextpos(p) - pusherror!(p, pos, pos, "numeral cannot end with an underscore") - nothing - else - p.charbuffer.ptr == 1 ? nothing : String(take!(p.charbuffer)) - end -end - -"Parses boolean" -function boolean(p::Parser, st::Int) - ch = '\x00' - - i = 0 - word = "true" - while !eof(p) && i0 - msec = Base.parse(Int, String(fsec)) - end - end - - # time zone - tzhour=0 - tzminute=0 - tzplus = true - tzminus = false - tzsign = true - if valid && !consume(p, 'Z') - tzplus = consume(p, '+') - if !tzplus - tzminus = consume(p, '-') - end - valid = valid && (tzplus || tzminus) - tzhour, valid = parsetwodigits(p, valid) - valid = valid && consume(p, ':') - tzminute, valid = parsetwodigits(p, valid) - - tzsign = tzplus - end - - if valid - dt = DateTime(year, month, day, - hour + (tzsign ? tzhour : -tzhour), - minute + (tzsign ? tzminute : -tzminute), - second, msec) - return dt - else - pusherror!(p, st, position(p), "malformed date literal") - return nothing - end -end - -"Parses a single or multi-line string" -function basicstring(p::Parser, st::Int) - !expect(p, '"') && return nothing - - multiline = false - - if consume(p, '"') - if consume(p, '"') - multiline = true - newline(p) - else - return "" - end - end - - basicstring(p, st, multiline) -end - -"Finish parsing a basic string after the opening quote has been seen" -function basicstring(p::Parser, st::Int, multiline::Bool) - while true - while multiline && newline(p) - write(p, '\n') - end - if eof(p) - pos = position(p) - pusherror!(p, st, pos, "unterminated string literal") - return nothing - else - ch = read(p) - if ch == '"' - if multiline - if !consume(p, '"') - write(p, '"') - continue - end - if !consume(p, '"') - write(p, '"') - write(p, '"') - continue - end - end - return String(take!(p.charbuffer)) - elseif ch == '\\' - pos = position(p) - ec = escape(p, pos, multiline) - ec !== nothing && write(p, ec) - elseif ch < '\x1f' - pos = position(p) - pusherror!(p, st, pos, "control character `$ch` must be escaped") - else - write(p, ch) - end - end - end -end - -"Reads character(s) after `\\` and transforms them into proper character" -function escape(p::Parser, st::Int, multiline::Bool) - if multiline && newline(p) - while whitespace(p) || newline(p) end - return nothing - end - pos = position(p) - if eof(p) - pusherror!(p, st, pos, "unterminated escape sequence") - return nothing - else - ch = read(p) - if ch == 'b' - '\b' - elseif ch == 't' - '\t' - elseif ch == 'n' - '\n' - elseif ch == 'f' - '\f' - elseif ch == 'r' - '\r' - elseif ch == '"' - '\"' - elseif ch == '\\' - '\\' - elseif ch == 'u' || ch == 'U' - len = ch == 'u' ? 4 : 8 - ucstr = ch == 'u' ? "\\u" : "\\U" - snum = String(read(p.input, len)) - try - if length(snum) < len - pusherror!(p, st, st+len, "expected $len hex digits after a `$ch` escape") - nothing - end - if !all(isxdigit, snum) - pusherror!(p, st, st+len, "unknown string escape: `$snum`") - nothing - end - c = unescape_string(ucstr * snum)[1] - c - catch - pusherror!(p, st, st+len, "codepoint `$snum` is not a valid unicode codepoint") - rewind(p, len) - nothing - end - else - escape_str = "\\x"*string(UInt32(ch), base=16, pad=2) - pusherror!(p, st, position(p), "unknown string escape: `$escape_str`") - nothing - end - end -end - -"Parses a single or multi-line literal string" -function literalstring(p::Parser, st::Int) - !expect(p, '\'') && return nothing - - multiline = false - - if consume(p, '\'') - if consume(p, '\'') - multiline = true - newline(p) - else - return "" - end - end - - literalstring(p, st, multiline) -end - -function literalstring(p::Parser, st::Int, multiline::Bool) - while true - if !multiline && newline(p) - npos = nextpos(p) - pusherror!(p, st, npos, "literal strings cannot contain newlines") - return nothing - end - if eof(p) - pusherror!(p, st, position(p), "unterminated string literal") - return nothing - else - ch = read(p.input, UInt8) - if ch == 0x27 - if multiline - if !consume(p, '\'') - write(p, 0x27) - continue - end - if !consume(p, '\'') - write(p, 0x27) - write(p, 0x27) - continue - end - end - return String(take!(p.charbuffer)) - else - write(p, ch) - end - end - end -end - -function array(p::Parser, st::Int) - !expect(p, '[') && return nothing - ret = Any[] - rettype = Any - expected = Any - while true - - # Break out early if we see the closing bracket - ignore(p) - consume(p, ']') && return ret - - # Attempt to parse a value, triggering an error if it's the wrong type. - pstart = nextpos(p) - npvalue = value(p) - npvalue === nothing && return nothing - pvalue = npvalue - - pend = nextpos(p) - valtype = isa(pvalue, Array) ? Array : typeof(pvalue) - expected = rettype === Any ? valtype : expected - if valtype != expected - pusherror!(p, pstart, pend, "expected type `$expected`, found type `$valtype`") - else - rettype = expected - push!(ret, pvalue) - end - - # Look for a comma. If we don't find one we're done - ignore(p) - !consume(p, ',') && break - end - ignore(p) - !expect(p, ']') && return nothing - return convert(Vector{rettype}, ret) -end - -function inlinetable(p::Parser, st::Int) - !expect(p, '{') && return nothing - whitespace(p) - - ret = Table(true) - consume(p, '}') && return ret - - while true - npos = nextpos(p) - k = keyname(p) - k === nothing && return nothing - !separator(p) && return nothing - v = value(p) - v === nothing && return nothing - insertpair(p, ret, k, v, npos) - - whitespace(p) - consume(p, '}') && break - !expect(p, ',') && return nothing - whitespace(p) - end - - return ret -end - -"Parses a value" -function value(p::Parser) - whitespace(p) - c = peek(p) - c === nothing && return nothing - ch = c - pos = position(p)+1 - if ch == '"' - return basicstring(p, pos) - elseif ch == '\'' - return literalstring(p, pos) - elseif ch == 't' || ch == 'f' - return boolean(p, pos) - elseif ch == '[' - return array(p, pos) - elseif ch == '{' - return inlinetable(p, pos) - elseif ch == '-' || ch == '+' || isdigit(ch) - return numdatetime(p, pos) - else - pusherror!(p, pos, pos+1, "expected a value") - return nothing - end -end - -"Parses the key-values and fills the given dictionary. Returns true in case of success and false in case of error." -function keyvalues(p::Parser, tbl::Table) - while true - whitespace(p) - newline(p) && continue - comment(p) && continue - nc = peek(p) - nc === nothing && break - nc == '[' && break - - # get key - klo = nextpos(p) - k = keyname(p) - k === nothing && return false - - # skip kv separator - !separator(p) && return false - - # get value - v = value(p) - v === nothing && return false - vend = nextpos(p) - - # insert kv into result table - insertpair(p, tbl, k, v, klo) - - whitespace(p) - if !comment(p) && !newline(p) - pc = peek(p) - pc === nothing && return true - pusherror!(p, klo, vend, "expected a newline after a key") - return false - end - end - return true -end - -"Construct nested structures from keys" -function nested(p, into, ks, kstart) - cnode = into - kend = 0 - for k in ks[1:end-1] - kend += length(k)+1 - if !haskey(cnode, k) - cnode.values[k] = Table(false) - cnode = cnode[k] - else - tmp = cnode[k] - if isa(tmp, Table) - cnode = tmp - elseif isa(tmp, Array) - if length(tmp)>0 && typeof(tmp[end]) === Table - cnode = tmp[end] - else - pusherror!(p, kstart, kstart+kend, "array `$k` does not contain tables") - return nothing, kend - end - else - pusherror!(p, kstart, kstart+kend, "key `$k` was not previously a table") - return nothing, kend - end - end - end - return cnode, kend -end - -function addtable(p::Parser, into::Table, ks::Vector{String}, tbl::Table, kstart) - - cnode, kend = nested(p, into, ks, kstart) - cnode === nothing && return - cur = cnode - - # fill last level with values - tkey = ks[end] - if haskey(cur, tkey) - ctbl = cur[tkey] - if isa(ctbl, Table) - ctbl.defined && pusherror!(p, kstart, kstart+kend+length(tkey), "redefinition of table `$tkey`") - - for (k,v) in tbl.values - insertpair(p, ctbl, k, v, kstart) - end - else - pusherror!(p, kstart, kstart+kend+length(tkey), "key `$tkey` was not previously a table") - end - else - cur.values[tkey] = tbl - end - -end - -function addarray(p::Parser, into::Table, ks::Vector{String}, val, kstart) - - cnode, kend = nested(p, into, ks, kstart) - cnode === nothing && return - cur = cnode - - akey = ks[end] - if haskey(cur, akey) - vec = cur[akey] - if isa(vec, Array) - if isa(val, eltype(vec)) - push!(vec, val) - else - pusherror!(p, kstart, kstart+kend+length(akey), "expected type `$(typeof(val))`, found type `$(eltype(vec))`") - end - else - pusherror!(p, kstart, kstart+kend+length(akey), "key `$akey` was previously not an array") - end - else - cur.values[akey] = Any[val] - end - -end - - -"""Executes the parser, parsing the string contained within. - -This function will return the `Table` instance if parsing is successful, or it will return `nothing` if any parse error or invalid TOML error occurs. - -If an error occurs, the `errors` field of this parser can be consulted to determine the cause of the parse failure. -""" -function parse(p::Parser) - ret = Table(false) - while !eof(p) - whitespace(p) - newline(p) && continue - comment(p) && continue - - # parse section - if consume(p, '[') - arr = consume(p, '[') - npos = nextpos(p) - - # parse the name of the section - ks = String[] - while true - whitespace(p) - s = keyname(p) - s !== nothing && push!(ks, s) - whitespace(p) - if consume(p, ']') - arr && !expect(p, ']') && return nothing - break - end - !expect(p, '.') && return nothing - end - isempty(ks) && return nothing - - # build the section - section = Table(true) - !keyvalues(p, section) && return nothing - if arr - addarray(p, ret, ks, section, npos) - else - addtable(p, ret, ks, section, npos) - end - else - !keyvalues(p, ret) && return nothing - end - end - - length(p.errors) != 0 && return nothing - - return ret -end diff --git a/ext/TOML/src/print.jl b/ext/TOML/src/print.jl deleted file mode 100644 index dffdcd226f..0000000000 --- a/ext/TOML/src/print.jl +++ /dev/null @@ -1,112 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -const TYPE = Union{AbstractDict,AbstractArray,AbstractString,DateTime,Bool} - -"Identify if character in subset of bare key symbols" -isbare(c::AbstractChar) = 'A' <= c <= 'Z' || 'a' <= c <= 'z' || isdigit(c) || c == '-' || c == '_' - -function printkey(io::IO, keys::Vector{String}) - for (i, k) in enumerate(keys) - i != 1 && Base.print(io, ".") - if length(k) == 0 - # empty key - Base.print(io, "\"\"") - elseif !all([isbare(c) for c in k]) - # quoted key - Base.print(io, "\"$(escape_string(k))\"") - else - Base.print(io, k) - end - end -end - -function printvalue(io::IO, value::AbstractArray; sorted=false) - Base.print(io, "[") - for (i, x) in enumerate(value) - i != 1 && Base.print(io, ", ") - if isa(x, AbstractDict) - _print(io, x, sorted=sorted) - else - printvalue(io, x, sorted=sorted) - end - end - Base.print(io, "]") -end -printvalue(io::IO, value::AbstractDict; sorted=false) = - _print(io, value, sorted=sorted) -printvalue(io::IO, value::DateTime; sorted=false) = - Base.print(io, Dates.format(value, dateformat"YYYY-mm-dd\THH:MM:SS.sss\Z")) -printvalue(io::IO, value::Bool; sorted=false) = - Base.print(io, value ? "true" : "false") -printvalue(io::IO, value::Integer; sorted=false) = - Base.print(io, Int(value)) # TOML specifies 64-bit signed long range for integer -printvalue(io::IO, value::AbstractFloat; sorted=false) = - Base.print(io, Float64(value)) # TOML specifies IEEE 754 binary64 for float -printvalue(io::IO, value; sorted=false) = - Base.print(io, "\"$(escape_string(string(value)))\"") - -is_table(value) = isa(value, AbstractDict) -is_array_of_tables(value) = isa(value, AbstractArray) && - length(value) > 0 && isa(value[1], AbstractDict) -is_tabular(value) = is_table(value) || is_array_of_tables(value) - -function _print(io::IO, a::AbstractDict, - ks::Vector{String} = String[]; - indent::Int = 0, - first_block::Bool = true, - sorted::Bool = false, - by = identity, -) - akeys = keys(a) - if sorted - akeys = sort!(collect(akeys), by = by) - end - - for key in akeys - value = a[key] - is_tabular(value) && continue - Base.print(io, ' '^4max(0,indent-1)) - printkey(io, [String(key)]) - Base.print(io, " = ") # print separator - printvalue(io, value, sorted = sorted) - Base.print(io, "\n") # new line? - first_block = false - end - - for key in akeys - value = a[key] - if is_table(value) - push!(ks, String(key)) - header = !all(is_tabular(v) for v in values(value))::Bool - if header - # print table - first_block || println(io) - first_block = false - Base.print(io, ' '^4indent) - Base.print(io,"[") - printkey(io, ks) - Base.print(io,"]\n") - end - # Use runtime dispatch here since the type of value seems not to be enforced other than as AbstractDict - Base.invokelatest(_print, io, value, ks; indent = indent + header, first_block = header, sorted = sorted, by = by) - pop!(ks) - elseif is_array_of_tables(value) - # print array of tables - first_block || println(io) - first_block = false - push!(ks, String(key)) - for v in value - Base.print(io, ' '^4indent) - Base.print(io,"[[") - printkey(io, ks) - Base.print(io,"]]\n") - !isa(v, AbstractDict) && error("array should contain only tables") - Base.invokelatest(_print, io, v, ks; indent = indent + 1, sorted = sorted, by = by) - end - pop!(ks) - end - end -end - -print(io::IO, a::AbstractDict; kwargs...) = _print(io, a; kwargs...) -print(a::AbstractDict; kwargs...) = print(stdout, a; kwargs...) diff --git a/ext/TOML/test/runtests.jl b/ext/TOML/test/runtests.jl deleted file mode 100644 index df3f3b09fc..0000000000 --- a/ext/TOML/test/runtests.jl +++ /dev/null @@ -1,690 +0,0 @@ -# This file is a part of Julia. License is MIT: https://julialang.org/license - -using TOML -import TOML: linecol, whitespace, comment, newline, expect, lookup, Parser, parse - - -using Test -using Dates - -macro testval(s, v) - f = "foo = $s" - :( @test TOML.parse(Parser($f))["foo"] == $v ) -end - -macro fail(s...) - teststr = s[1] - errstr = "" - debug = false - if length(s) == 2 - if isa(s[2], String) - errstr = s[2] - elseif isa(s[2], Expr) - debug = s[2].args[1] == :debug - end - end - if length(s) == 3 - errstr = s[2] - debug = s[3].args[1] == :debug - end - - # macro vars - pvar = :pv - ppvar = :ppv - - # debugging report - dbgexp = if debug - quote - println("\nTEST FAIL: ", escape_string($teststr)) - if length($pvar.errors) > 0 - println("ERRORS:") - for e in $pvar.errors - println(e) - end - end - if $ppvar !== nothing - println("RESULT:") - println(TOML.$ppvar) - end - end - else - :() - end - - # error for comparison - errtsts = if !isempty(errstr) - quote - @test length($pvar.errors)>0 - @test findlast(m->m==$errstr, map(e->e.msg, $pvar.errors)) != 0 - end - else - :() - end - - quote - local $pvar = Parser($teststr) - local $ppvar = parse($pvar) - $dbgexp - $errtsts - @test $ppvar === nothing - end -end - -macro success(s...) - teststr = s[1] - debug = false - if length(s) == 2 && isa(s[2], Expr) - debug = s[2].args[1] == :debug - end - - # macro vars - pvar = :pv - ppvar = :ppv - - # debugging report - dbgexp = if debug - quote - println("\nTEST SUCCESS: ", escape_string($teststr)) - if length($pvar.errors) > 0 - println("ERRORS:") - for e in $pvar.errors - println(e) - end - end - if $ppvar !== nothing - println("RESULT:") - println(TOML.$ppvar) - end - end - else - :() - end - - quote - local $pvar = Parser($teststr) - local $ppvar = parse($pvar) - $dbgexp - @test $ppvar !== nothing - end -end - -@testset "TOML" begin -@testset "Parser" begin - - @testset "Parser internal functions" begin - test = """ - #[test]\r - [test] - foo = "bar" -""" - p = Parser(test) - @test whitespace(p) - @test position(p) == 4 - @test comment(p) - @test position(p) == 13 - @test whitespace(p) - @test position(p) == 16 - @test !expect(p, 'a') - @test expect(p, '[') - @test position(p) == 17 - end - - - @testset "Lookups" begin - p = Parser("""hello."world\\t".a.0.'escaped'.value""") - @testset for (p, s) in zip(TOML.lookup(p), ["hello"; "world\t"; "a"; "0"; "escaped"; "value"]) - @test p == s - end - - p = Parser("") - @test TOML.lookup(p) == String[] - - p = Parser("value") - @test TOML.lookup(p) == String["value"] - - p = Parser("\"\"") - #TODO: @test get(lookup(p)) == String[""] - - end - - @testset "New line" begin - @success(""" -[project.A]\r -name = \"splay2\"\r -\r -[project]\r -\r -name = \"splay\"\r -version = \"0.1.0\"\r -authors = [\"alex@crichton.co\"]\r -\r -[[lib]]\r -\r -path = \"lib.rs\"\r -name = \"splay\"\r -description = \"\"\" -A Rust implementation of a TAR file reader and writer. This library does not\r -currently handle compression, but it is abstract over all I/O readers and\r -writers. Additionally, great lengths are taken to ensure that the entire\r -contents are never required to be entirely resident in memory all at once.\r -\"\"\" -[[lib.aaa]]\r -bbb = \"aaa\"\r -""") - - @fail("\r") - - @fail("a = [ \r ]") - - @fail("a = \"\"\"\r\"\"\"") - - @fail("a = \"\"\"\\ \r \"\"\"") - - @success("a = '''\r'''") - - @success("a = '\r'") - - @fail("a = \"\n\"") - - @fail("a = '\n'") - - p = Parser(" - foo = \"\"\"\\\r\n\"\"\" - bar = \"\"\"\\\r\n \r\n \r\n a\"\"\" - ") - res = TOML.parse(p) - @test res["foo"] == "" - @test res["bar"] == "a" - - @fail("0=0r=false", "expected a newline after a key") - - @fail(""" -0=""o=""m=""r=""00="0"q=\"\"\"0\"\"\"e=\"\"\"0\"\"\" -""", "expected a newline after a key") - - @fail(""" -[[0000l0]] -0="0"[[0000l0]] -0="0"[[0000l0]] -0="0"l="0" -""", "expected a newline after a key") - - @fail(""" -0=[0]00=[0,0,0]t=["0","0","0"]s=[1000-00-00T00:00:00Z,2000-00-00T00:00:00Z] -""", "expected a newline after a key") - - @fail(""" -0=0r0=0r=false -""", "expected a newline after a key") - - @fail(""" -0=0r0=0r=falsefal=false -""", "expected a newline after a key") - - end - - @testset "Offset->(Line, Col) " begin - p = Parser("ab\ncde\nf") - @test linecol(p, 0) == (1,0) - @test linecol(p, 1) == (1,1) - @test linecol(p, 4) == (2,1) - @test linecol(p, 5) == (2,2) - @test linecol(p, 8) == (3,1) - @test linecol(p, 9) == (3,0) - end - - @testset "Strings" begin - p = Parser(""" -bar = "\\U00000000" -key1 = "One\\nTwo" -key2 = \"\"\"One\nTwo\"\"\" -key3 = \"\"\" -One -Two\"\"\" - -key4 = "The quick brown fox jumps over the lazy dog." -key5 = \"\"\" -The quick brown \\ - - - fox jumps over \\ - the lazy dog.\"\"\" -key6 = \"\"\"\\ - The quick brown \\ - fox jumps over \\ - the lazy dog.\\ - \"\"\" - -# What you see is what you get. -winpath = 'C:\\Users\\nodejs\\templates' -winpath2 = '\\\\ServerX\\admin\$\\system32\\' -quoted = 'Tom "Dubs" Preston-Werner' -regex = '<\\i\\c*\\s*>' -regex2 = '''I [dw]on't need \\d{2} apples''' -lines = ''' -The first newline is -trimmed in raw strings. - All other whitespace - is preserved. -''' -""") - res = TOML.parse(p) - - @test res["bar"] == "\0" - @test res["key1"] == "One\nTwo" - @test res["key2"] == "One\nTwo" - @test res["key3"] == "One\nTwo" - - msg = "The quick brown fox jumps over the lazy dog." - @test res["key4"] == msg - @test res["key5"] == msg - @test res["key6"] == msg - - @test res["winpath"] == "C:\\Users\\nodejs\\templates" - @test res["winpath2"] == "\\\\ServerX\\admin\$\\system32\\" - @test res["quoted"] == """Tom "Dubs" Preston-Werner""" - @test res["regex"] == """<\\i\\c*\\s*>""" - @test res["regex2"] == """I [dw]on't need \\d{2} apples""" - - @test res["lines"] == "The first newline is\ntrimmed in raw strings.\n All other whitespace\n is preserved.\n" - - - @testval("''", "") - @testval("\"\"", "") - @testval("\"\"\"\n\n\n\"\"\"", "\n\n") - - #@fail("foo = \"\\uD800\"") - - @fail("foo = \"\\uxx\"") - @fail("foo = \"\\u\"") - @fail("foo = \"\\") - @fail("foo = '") - - end - - @testset "Numbers" begin - @fail("a = 00") - @fail("a = -00") - @fail("a = +00") - @fail("a = 00.0") - @fail("a = -00.0") - @fail("a = +00.0") - @fail("a = 9223372036854775808") - @fail("a = -9223372036854775809") - - @fail("a = 0.") - @fail("a = 0.e") - @fail("a = 0.E") - @fail("a = 0.0E") - @fail("a = 0.0e") - @fail("a = 0.0e-") - @fail("a = 0.0e+") - @fail("a = 0.0e+00") - - @testval("1.0", 1.0) - @testval("1.0e0", 1.0) - @testval("1.0e+0", 1.0) - @testval("1.0e-0", 1.0) - @testval("1.001e-0", 1.001) - @testval("2e10", 2e10) - @testval("2e+10", 2e10) - @testval("2e-10", 2e-10) - @testval("2_0.0", 20.0) - @testval("2_0.0_0e0_0", 20.0) - @testval("2_0.1_0e1_0", 20.1e10) - - @fail("4") - - @testval("1_0", 10) - @testval("1_0_0", 100) - @testval("1_000", 1000) - @testval("+1_000", 1000) - @testval("-1_000", -1000) - - @fail("foo = 0_") - @fail("foo = 0__0") - @fail("foo = __0") - @fail("foo = 1_0_") - - end - - @testset "Booleans" begin - - @testval("true", true) - @testval("false", false) - - @fail("foo = true2") - @fail("foo = false2") - @fail("foo = t1") - @fail("foo = f1") - - end - - @testset "Datetime" begin - - @testval("2016-09-09T09:09:09Z", DateTime(2016,9,9,9,9,9)) - @testval("2016-09-09T09:09:09.0Z", DateTime(2016,9,9,9,9,9)) - @testval("2016-09-09T09:09:09.0+10:00", DateTime(2016,9,9,19,9,9)) - @testval("2016-09-09T09:09:09.012-02:00", DateTime(2016,9,9,7,9,9,12)) - @testval("2016-09-09T09:09:09.0+10:00", Dates.DateTime(2016,9,9,19,9,9)) - @testval("2016-09-09T09:09:09.012-02:00", Dates.DateTime(2016,9,9,7,9,9,12)) - - @fail("foo = 2016-09-09T09:09:09.Z", "malformed date literal") - @fail("foo = 2016-9-09T09:09:09Z", "malformed date literal") - @fail("foo = 2016-09-09T09:09:09+2:00", "malformed date literal") - @fail("foo = 2016-09-09T09:09:09-2:00", "malformed date literal") - @fail("foo = 2016-09-09T09:09:09Z-2:00", "expected a newline after a key") - @fail("foo = 2016-09-09s09:09:09Z", "malformed date literal") - @fail("foo = 2016-09-09 09:09:09x", "malformed date literal") - - end - - - @testset "Keys" begin - - p = Parser(" - foo = 3 - foo_3 = 3 - foo_-2--3--r23f--4-f2-4 = 3 - _ = 3 - - = 3 - 8 = 8 - \"a\" = 3 - \"!\" = 3 - \"a^b\" = 3 - \"\\\\\\\"\" = 3 - \"character encoding\" = \"value\" - 'ʎǝʞ' = \"value\" - ") - res = TOML.parse(p) - - @test haskey(res, "foo") - @test haskey(res, "-") - @test haskey(res, "_") - @test haskey(res, "8") - @test haskey(res, "foo_3") - @test haskey(res, "foo_-2--3--r23f--4-f2-4") - @test haskey(res, "a") - @test haskey(res, "!") - @test haskey(res, "\\\"") - @test haskey(res, "character encoding") - @test haskey(res, "ʎǝʞ") - - @fail("key\n=3") - @fail("key=\n3") - @fail("key|=3") - @fail("\"\"=3") - @fail("=3") - @fail("\"\"|=3") - @fail("\"\n\"|=3") - @fail("\"\r\"|=3") - - end - - - @testset "Tables" begin - - @fail("[]") - @fail("[.]") - @fail("[\"\".\"\"]") - @fail("[a.]") - @fail("[\"\"]") - @fail("[!]") - @fail("[\"\n\"]") - @fail("[a.b]\n[a.\"b\"]") - @fail("[']") - @fail("[''']") - @fail("['''''']") - @fail("['\n']") - @fail("['\r\n']") - - p = Parser(" - [a.\"b\"] - [\"f f\"] - [\"f.f\"] - [\"\\\\\\\"\"] - ['a.a'] - ['\"\"'] - ") - res = TOML.parse(p) - @test haskey(res, "a.a") - @test haskey(res, "f f") - @test haskey(res, "f.f") - @test haskey(res, "\\\"") - @test haskey(res, "\"\"") - @test haskey(res["a"], "b") - - - @test haskey(TOML.parse(Parser("[foo]")), "foo") - - - @testset "Inline Tables" begin - - @success("a = {}") - @success("a = {b=1}") - @success("a = { b = 1 }") - @success("a = {a=1,b=2}") - @success("a = {a=1,b=2,c={}}") - @fail("a = {a=1,}") - @fail("a = {,}") - @fail("a = {a=1,a=1}") - @fail("a = {\n}") - @fail("a = {") - @success("a = {a=[\n]}") - @success("a = {\"a\"=[\n]}") - @success("a = [\n{},\n{},\n]") - - end - - - @testset "Redefinition" begin - @fail(" - [a] - foo=\"bar\" - [a.b] - foo=\"bar\" - [a] - ", "redefinition of table `a`") - @fail(" - [a] - foo=\"bar\" - b = { foo = \"bar\" } - [a] - ", "redefinition of table `a`") - @success(" - [a.b] - c = {} - [a] - d = 1 - ") - @fail(" - [a.b] - c = 1 - [a] - b = {} - ", "duplicate key `b`") - @fail(" - [a] - b = {} - [a] - ", "redefinition of table `a`") - end - - @testset "Nesting" begin - @fail(" - a = [2] - [[a]] - b = 5 - ", "expected type `TOML.Table`, found type `$(Int)`") - @fail(" - a = 1 - [a.b] - ", "key `a` was not previously a table") - @fail(" - a = [] - [a.b] - ", "array `a` does not contain tables") - @success(" - a = [{}] - [a.b] - ") - @fail(" - a = [] - [[a.b]] - ", "array `a` does not contain tables") - @fail(" - [a] - b = { c = 2, d = {} } - [a.b] - c = 2 - ", "duplicate key `c`") - end - end - - @testset "Arrays" begin - - p = Parser(""" -[[foo]] - #… - [foo.bar] - #… -[[foo]] # ... - #… - [foo.bar] - #... -""") - res = parse(p) - @test haskey(res, "foo") - arr = res["foo"] - @test length(arr) == 2 - @test isa(arr[1], TOML.Table) - @test haskey(arr[1], "bar") - @test haskey(arr[2], "bar") - - p = Parser(""" -[[fruit]] - name = "apple" - [fruit.physical] - color = "red" - shape = "round" - [[fruit.variety]] - name = "red delicious" - [[fruit.variety]] - name = "granny smith" -[[fruit]] - name = "banana" - [[fruit.variety]] - name = "plantain" -""") - res = parse(p) - @test haskey(res, "fruit") - fruit = res["fruit"] - - @test length(fruit) == 2 - @test isa(fruit[1], TOML.Table) - apple = fruit[1] - banana = fruit[2] - - @test haskey(apple, "name") - @test apple["name"] == "apple" - @test haskey(apple, "physical") - @test apple["physical"]["color"] == "red" - @test apple["physical"]["shape"] == "round" - @test apple["variety"][1]["name"] == "red delicious" - @test apple["variety"][2]["name"] == "granny smith" - - @test haskey(banana, "name") - @test banana["name"] == "banana" - @test banana["variety"][1]["name"] == "plantain" - - res = TOML.parse(""" -[[products]] -name = "Hammer" -sku = 738594937 - -[[products]] - -[[products]] -name = "Nail" -sku = 284758393 -color = "gray" -""") - @test haskey(res, "products") - products = res["products"] - @test isa(products, Array) - @test length(products) == 3 - @test products[1]["name"] == "Hammer" - @test products[1]["sku"] == 738594937 - @test length(products[2]) == 0 - @test products[3]["name"] == "Nail" - @test products[3]["sku"] == 284758393 - @test products[3]["color"] == "gray" - - @fail("""array = [ - # "This might most likely happen in multiline arrays", - # Like here, - # "or here, - # and here" - # ] End of array comment, forgot the # - """) - end - -end - -@testset "Printer" begin - res1 = TOML.parse(""" -title = "TOML Example" -[owner] -bio = \"\"\"GitHub Cofounder & CEO -Likes tater tots and beer.\"\"\" -dob = 1979-05-27T07:32:00Z # First class dates? Why not? - -[database] -server = "192.168.1.1" -ports = [ 8001, 8001, 8002 ] -enabled = true - -[servers] - - # You can indent as you please. Tabs or spaces. TOML don't care. - [servers.alpha] - ip = "10.0.0.1" - dc = "eqdc10" - - [servers.beta] - ip = "10.0.0.2" - dc = "eqdc10" - country = "中国" # This should be parsed as UTF-8 - -[clients] -data = [ ["gamma", "delta"], [1, 2] ] # just an update to make sure parsers support it - - [[products]] - name = "Hammer" - sku = 738594937 - - [[products]] - name = "Nail" - sku = 284758393 - color = "gray" -""") - io = IOBuffer() - TOML.print(io, res1) - seekstart(io) - res2 = TOML.parse(io) - @test res1 == res2 - - # sorting with non-Function by - io = IOBuffer() - d = Dict("1.10" => 10, "1.1" => 1) - TOML.print(io, d; sorted=true, by = VersionNumber) - @test String(take!(io)) == "\"1.1\" = 1\n\"1.10\" = 10\n" - - # symbols as Keys - io = IOBuffer() - d = Dict(:products=>[Dict("name"=>"me","cnt"=>3),Dict{String,Any}(),Dict(:name=>"you","cnt"=>5)]) - TOML.print(io, d) - @test String(take!(io)) == "[[products]]\nname = \"me\"\ncnt = 3\n[[products]]\n[[products]]\ncnt = 5\nname = \"you\"\n" - -end -end diff --git a/src/API.jl b/src/API.jl index 85cf9601cf..e7ea8c7a0e 100644 --- a/src/API.jl +++ b/src/API.jl @@ -372,7 +372,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), verbose=false, return end - for (filename, infos) in TOML.parse(String(read(usage_filepath))) + for (filename, infos) in TOML.parsefile(usage_filepath) f.(Ref(filename), infos) end end diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 994f1716cc..61d486518e 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -630,7 +630,7 @@ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; end end else - artifact_dict = TOML.DictType() + artifact_dict = Dict{String, Any}() end # Otherwise, the new piece of data we're going to write out is this dict: diff --git a/src/Operations.jl b/src/Operations.jl index a400b80a54..8defb1424e 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -157,12 +157,12 @@ end #################### function load_versions(ctx, path::String; include_yanked=false) - toml = parse_toml(path, "Versions.toml"; fakeit=true) + toml = parse_toml(joinpath(path, "Versions.toml"); fakeit=true) versions = Dict{VersionNumber, SHA1}( VersionNumber(ver) => SHA1(info["git-tree-sha1"]) for (ver, info) in toml if !get(info, "yanked", false) || include_yanked) if Pkg.OFFLINE_MODE[] # filter out all versions that are not already downloaded - pkg = parse_toml(path, "Package.toml") + pkg = parse_toml(joinpath(path, "Package.toml")) filter!(versions) do (v, sha) pkg_spec = PackageSpec(name=pkg["name"], uuid=UUID(pkg["uuid"]), version=v, tree_hash=sha) return is_package_downloaded(ctx, pkg_spec) @@ -506,7 +506,7 @@ function load_urls(ctx::Context, pkgs::Vector{PackageSpec}) ver = pkg.version::VersionNumber urls[uuid] = String[] for path in registered_paths(ctx, uuid) - info = parse_toml(path, "Package.toml") + info = parse_toml(joinpath(path, "Package.toml")) repo = info["repo"] repo in urls[uuid] || push!(urls[uuid], repo) end @@ -1624,7 +1624,7 @@ function stat_rep(x::PackageSpec; name=true) if x.repo.rev !== nothing rev = occursin(r"\b([a-f0-9]{40})\b", x.repo.rev) ? x.repo.rev[1:7] : x.repo.rev end - subdir_str = x.repo.subdir == nothing ? "" : ":$(x.repo.subdir)" + subdir_str = x.repo.subdir === nothing ? "" : ":$(x.repo.subdir)" repo = Operations.is_tracking_repo(x) ? "`$(x.repo.source)$(subdir_str)#$(rev)`" : "" path = Operations.is_tracking_path(x) ? "$(pathrepr(x.path))" : "" pinned = x.pinned ? "⚲" : "" diff --git a/src/Pkg.jl b/src/Pkg.jl index 472c0c8f10..4ca6560fae 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -4,6 +4,7 @@ module Pkg import Random import REPL +import TOML export @pkg_str export PackageSpec @@ -40,9 +41,6 @@ const UPDATED_REGISTRY_THIS_SESSION = Ref(false) const OFFLINE_MODE = Ref(false) const DEFAULT_IO = Ref{Union{Nothing,IO}}(nothing) -# load snapshotted dependencies -include("../ext/TOML/src/TOML.jl") - include("utils.jl") include("GitTools.jl") include("PlatformEngines.jl") @@ -591,7 +589,7 @@ function _run_precompilation_script_setup() touch("Project.toml") Pkg.activate(".") Pkg.generate("TestPkg") - uuid = Pkg.TOML.parsefile(joinpath("TestPkg", "Project.toml"))["uuid"] + uuid = TOML.parsefile(joinpath("TestPkg", "Project.toml"))["uuid"] mv("TestPkg", "TestPkg.jl") tree_hash = cd("TestPkg.jl") do sig = LibGit2.Signature("TEST", "TEST@TEST.COM", round(time()), 0) diff --git a/src/Types.jl b/src/Types.jl index 396159c840..64538fb6f7 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -10,7 +10,7 @@ import REPL import Base.string using REPL.TerminalMenus -using ..TOML +using TOML import ...Pkg, ..UPDATED_REGISTRY_THIS_SESSION, ..DEFAULT_IO import ...Pkg: GitTools, depots, depots1, logdir, set_readonly, safe_realpath, pkg_server import ..BinaryPlatforms: Platform @@ -149,11 +149,6 @@ end # EnvCache # ############ -function parse_toml(path::String...; fakeit::Bool=false) - p = joinpath(path...) - !fakeit || isfile(p) ? TOML.parsefile(p) : Dict{String,Any}() -end - function projectfile_path(env_path::String; strict=false) for name in Base.project_names maybe_file = joinpath(env_path, name) @@ -338,6 +333,7 @@ Base.@kwdef mutable struct Context currently_running_target::Bool = false # test instrumenting status_io::Union{IO,Nothing} = nothing + parser::TOML.Parser = TOML.Parser() end project_uuid(ctx::Context) = ctx.env.pkg === nothing ? nothing : ctx.env.pkg.uuid @@ -1246,7 +1242,7 @@ function find_registered!(ctx::Context, for registry in collect_registries() reg_abspath = abspath(registry.path) data = read_registry(joinpath(registry.path, "Registry.toml")) - for (_uuid::String, pkgdata::TOML.DictType) in data["packages"] + for (_uuid::String, pkgdata::Dict{String, Any}) in data["packages"] name = pkgdata["name"]::String uuid = UUID(_uuid) push!(get!(Vector{UUID}, ctx.env.uuids, name), uuid) @@ -1336,7 +1332,7 @@ function registered_info(ctx::Context, uuid::UUID, key::String) isempty(paths) && pkgerror("`$uuid` is not registered") values = [] for path in paths - info = parse_toml(path, "Package.toml") + info = parse_toml(joinpath(path, "Package.toml")) value = get(info, key, nothing) push!(values, (path, value)) end @@ -1417,4 +1413,12 @@ Base.@kwdef struct ProjectInfo path::String end +# TOML stuff + +parse_toml(ctx::Context, path::String; fakeit::Bool=false) = + parse_toml(ctx.parser, path; fakeit) +parse_toml(path::String; fakeit::Bool=false) = parse_toml(TOML.Parser(), path; fakeit) +parse_toml(parser::TOML.Parser, path::String; fakeit::Bool=false) = + !fakeit || isfile(path) ? TOML.parsefile(parser, path) : Dict{String,Any}() + end # module diff --git a/src/manifest.jl b/src/manifest.jl index ef35a7babe..1a1dae2e0f 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -158,25 +158,14 @@ function Manifest(raw::Dict)::Manifest return validate_manifest(stage1) end -function read_manifest(path_or_stream::Union{String,IO}) - local raw - try - if path_or_stream isa String - path = path_or_stream - isfile(path) || return Dict{UUID,PackageEntry}() - raw = TOML.parsefile(path) - else - raw = TOML.parse(path_or_stream) - end - catch err - path = path_or_stream isa String ? path_or_stream : "" - if err isa TOML.ParserError - pkgerror("Could not parse manifest $path: $(err.msg)") - elseif isa(err, CompositeException) && all(x -> x isa TOML.ParserError, err) - pkgerror("Could not parse manifest $path: $err") - else - rethrow() - end +function read_manifest(f_or_io::Union{String,IO}) + raw = if f_or_io isa IO + TOML.tryparse(read(f_or_io, String)) + else + isfile(f_or_io) ? TOML.tryparsefile(f_or_io) : return Dict{UUID,PackageEntry}() + end + if raw isa TOML.ParserError + pkgerror("Could not parse manifest: ", sprint(showerror, raw)) end return Manifest(raw) end @@ -189,7 +178,7 @@ function destructure(manifest::Manifest)::Dict if value == default delete!(entry, key) else - entry[key] = value isa TOML.TYPE ? value : string(value) + entry[key] = value end end @@ -237,8 +226,12 @@ end write_manifest(manifest::Manifest, manifest_file::AbstractString) = write_manifest(destructure(manifest), manifest_file) function write_manifest(manifest::Dict, manifest_file::AbstractString) - io = IOBuffer() - print(io, "# This file is machine-generated - editing it directly is not advised\n\n") - TOML.print(io, manifest, sorted=true) - open(f -> write(f, seekstart(io)), manifest_file; truncate=true) + str = sprint() do io + print(io, "# This file is machine-generated - editing it directly is not advised\n\n") + TOML.print(io, manifest, sorted=true) do x + (x isa UUID || x isa SHA1 || x isa VersionNumber) && return string(x) + error("unhandled type `$(typeof(x))`") + end + end + write(manifest_file, str) end diff --git a/src/project.jl b/src/project.jl index 82d3efda63..4340920885 100644 --- a/src/project.jl +++ b/src/project.jl @@ -124,24 +124,18 @@ function Project(raw::Dict) return project end -function read_project(io::IO; path=nothing) - raw = nothing - try - raw = TOML.parse(io) - catch err - if err isa TOML.ParserError - pkgerror("Could not parse project $(something(path,"")): $(err.msg)") - elseif err isa CompositeException && all(x -> x isa TOML.ParserError, err) - pkgerror("Could not parse project $(something(path,"")): $err") - else - rethrow() - end +function read_project(f_or_io::Union{String, IO}) + raw = if f_or_io isa IO + TOML.tryparse(read(f_or_io, String)) + else + isfile(f_or_io) ? TOML.tryparsefile(f_or_io) : return Project() + end + if raw isa TOML.ParserError + pkgerror("Could not parse project: ", sprint(showerror, raw)) end return Project(raw) end -read_project(path::String) = - isfile(path) ? open(io->read_project(io;path=path), path) : Project() ########### # WRITING # @@ -175,7 +169,11 @@ end write_project(project::Project, project_file::AbstractString) = write_project(destructure(project), project_file) function write_project(project::Dict, project_file::AbstractString) - io = IOBuffer() - TOML.print(io, project, sorted=true, by=key -> (project_key_order(key), key)) - open(f -> write(f, seekstart(io)), project_file; truncate=true) + str = sprint() do io + TOML.print(io, project, sorted=true, by=key -> (project_key_order(key), key)) do x + (x isa UUID || x isa VersionNumber) && return string(x) + error("unhandled type `$(typeof(x))`") + end + end + write(project_file, str) end diff --git a/test/artifacts.jl b/test/artifacts.jl index 8fecd0ba53..57c3cfac45 100644 --- a/test/artifacts.jl +++ b/test/artifacts.jl @@ -3,7 +3,7 @@ import ..Pkg # ensure we are using the correct Pkg using Test, Random, Pkg.Artifacts, Pkg.BinaryPlatforms, Pkg.PlatformEngines import Pkg.Artifacts: pack_platform!, unpack_platform, with_artifacts_directory, ensure_all_artifacts_installed, extract_all_hashes -using Pkg.TOML, Dates +using TOML, Dates import Base: SHA1 using ..Utils @@ -248,7 +248,7 @@ end @test ensure_artifact_installed("foo_txt", artifacts_toml) == artifact_path(hash) # Test that binding caused an entry in the manifest_usage.toml - usage = Pkg.TOML.parse(String(read(joinpath(Pkg.logdir(), "artifact_usage.toml")))) + usage = TOML.parsefile(joinpath(Pkg.logdir(), "artifact_usage.toml")) @test any(x -> startswith(x, artifacts_toml), keys(usage)) # Test that we can overwrite bindings @@ -436,7 +436,7 @@ end bind_artifact!(artifacts_toml, "die", die_hash) # Now test that the usage file exists, and contains our Artifacts.toml - usage = Pkg.TOML.parse(String(read(usage_path))) + usage = TOML.parsefile(usage_path) @test any(x -> startswith(x, artifacts_toml), keys(usage)) # Test that a gc() doesn't remove anything @@ -454,7 +454,7 @@ end @test artifact_exists(die_hash) orphaned_path = joinpath(Pkg.logdir(), "orphaned.toml") - orphanage = Pkg.TOML.parse(String(read(orphaned_path))) + orphanage = TOML.parsefile(orphaned_path) @test any(x -> startswith(x, artifact_path(die_hash)), keys(orphanage)) # Now, sleep for 0.2 seconds, then gc with a collect delay of 0.1 seconds @@ -467,10 +467,10 @@ end # die_hash should still be listed within the orphan list, but one more gc() will # remove it; this is intentional and allows for robust removal scheduling. - orphanage = Pkg.TOML.parse(String(read(orphaned_path))) + orphanage = TOML.parsefile(orphaned_path) @test any(x -> startswith(x, artifact_path(die_hash)), keys(orphanage)) Pkg.gc() - orphanage = Pkg.TOML.parse(String(read(orphaned_path))) + orphanage = TOML.parsefile(orphaned_path) @test !any(x -> startswith(x, artifact_path(die_hash)), keys(orphanage)) # Next, unbind the live_hash, then run with collect_delay=0, and ensure that diff --git a/test/pkg.jl b/test/pkg.jl index fc16608033..1f0ec28608 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -8,6 +8,7 @@ import LibGit2 using Test using UUIDs using Dates +using TOML using Pkg using Pkg.Types @@ -357,7 +358,7 @@ temp_pkg_dir() do project_path end @testset "check logging" begin - usage = Pkg.TOML.parse(String(read(joinpath(Pkg.logdir(), "manifest_usage.toml")))) + usage = TOML.parsefile(joinpath(Pkg.logdir(), "manifest_usage.toml")) manifest = Pkg.safe_realpath(joinpath(project_path, "Manifest.toml")) @test any(x -> startswith(x, manifest), keys(usage)) end diff --git a/test/repl.jl b/test/repl.jl index ea1a1661fb..de7855bdf7 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -8,6 +8,7 @@ using Pkg.Types: manifest_info, EnvCache, Context import Pkg.Types.PkgError using UUIDs using Test +using TOML import LibGit2 using ..Utils @@ -551,10 +552,10 @@ end @testset "unit test for REPLMode.promptf" begin function set_name(projfile_path, newname) sleep(1.1) - project = Pkg.TOML.parsefile(projfile_path) + project = TOML.parsefile(projfile_path) project["name"] = newname open(projfile_path, "w") do io - Pkg.TOML.print(io, project) + TOML.print(io, project) end end diff --git a/test/test_packages/BuildProjectFixedDeps/Manifest.toml b/test/test_packages/BuildProjectFixedDeps/Manifest.toml index 4a1babbc46..02291c6606 100644 --- a/test/test_packages/BuildProjectFixedDeps/Manifest.toml +++ b/test/test_packages/BuildProjectFixedDeps/Manifest.toml @@ -28,6 +28,7 @@ uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.19.0" [[LibGit2]] +deps = ["Printf"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[Libdl]] @@ -48,7 +49,7 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[Pkg]] -deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[Printf]] @@ -56,7 +57,7 @@ deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" [[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] @@ -80,6 +81,10 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" deps = ["LinearAlgebra", "SparseArrays"] uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +[[TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" + [[Test]] deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/test_packages/BuildProjectFixedDeps/Project.toml b/test/test_packages/BuildProjectFixedDeps/Project.toml index 93f9a83e41..84de310b4a 100644 --- a/test/test_packages/BuildProjectFixedDeps/Project.toml +++ b/test/test_packages/BuildProjectFixedDeps/Project.toml @@ -5,3 +5,4 @@ version = "0.1.0" [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" diff --git a/test/test_packages/BuildProjectFixedDeps/deps/build.jl b/test/test_packages/BuildProjectFixedDeps/deps/build.jl index b993b221c9..f592e8f7c8 100644 --- a/test/test_packages/BuildProjectFixedDeps/deps/build.jl +++ b/test/test_packages/BuildProjectFixedDeps/deps/build.jl @@ -1,4 +1,4 @@ -import Pkg.TOML +import TOML build_artifact = joinpath(@__DIR__, "artifact") isfile(build_artifact) && rm(build_artifact) project = TOML.parsefile(Base.active_project()) diff --git a/test/utils.jl b/test/utils.jl index fdea58cb0d..ccc3152ad5 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -3,6 +3,7 @@ module Utils import ..Pkg +using TOML export temp_pkg_dir, cd_tempdir, isinstalled, write_build, with_current_env, with_temp_env, with_pkg_env, git_init_and_commit, copy_test_package, @@ -215,7 +216,7 @@ function copy_test_package(tmpdir::String, name::String; use_pkg=true) # The known Pkg UUID, and whatever UUID we're currently using for testing known_pkg_uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - pkg_uuid = Pkg.TOML.parsefile(joinpath(dirname(@__DIR__), "Project.toml"))["uuid"] + pkg_uuid = TOML.parsefile(joinpath(dirname(@__DIR__), "Project.toml"))["uuid"] # We usually want this test package to load our pkg, so update its Pkg UUID: test_pkg_dir = joinpath(@__DIR__, "test_packages", name) @@ -230,7 +231,7 @@ end function add_this_pkg() pkg_dir = dirname(@__DIR__) - pkg_uuid = Pkg.TOML.parsefile(joinpath(pkg_dir, "Project.toml"))["uuid"] + pkg_uuid = TOML.parsefile(joinpath(pkg_dir, "Project.toml"))["uuid"] spec = Pkg.PackageSpec( name="Pkg", uuid=UUID(pkg_uuid),