From 97b2da6d7b6a3ecb5cca1328fd146523e44fd976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Fri, 19 Feb 2021 22:24:08 -0500 Subject: [PATCH 01/13] =?UTF-8?q?=E2=9C=A8=20ASCII=20(Fixes=20#26)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/src/man/data.md | 6 ++++ src/datasets/ascii.jl | 69 +++++++++++++++++++++++++++++++++++++++++++ test/ascii.jl | 24 +++++++++++++++ test/runtests.jl | 1 + 4 files changed, 100 insertions(+) create mode 100644 src/datasets/ascii.jl create mode 100644 test/ascii.jl diff --git a/docs/src/man/data.md b/docs/src/man/data.md index 926a9756..f4b95141 100644 --- a/docs/src/man/data.md +++ b/docs/src/man/data.md @@ -21,3 +21,9 @@ bioclim ```@docs landcover ``` + +## ASCII files + +```@docs +ascii +``` \ No newline at end of file diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl new file mode 100644 index 00000000..7deab90c --- /dev/null +++ b/src/datasets/ascii.jl @@ -0,0 +1,69 @@ +function _get_asc_field(lines, field, type) + field_line = first(filter(startswith(field), lines)) + return parse(type, last(split(field_line, " "))), findfirst(startswith(field), lines) +end + +""" + ascii(file::AbstractString, datatype::Type{T}=Float64) where {T <: Number} + +Reads the content of a grid file to a `SimpleSDMPredictor`, the type of which is +given by the `datatype` argument. +""" +function ascii(file::AbstractString, datatype::Type{T}=Float64) where {T <: Number} + lines = lowercase.(readlines(file)) + # Get the information + ncols, ncols_line = _get_asc_field(lines, "ncols", Int64) + nrows, nrows_line = _get_asc_field(lines, "nrows", Int64) + xl, xl_line = _get_asc_field(lines, "xllcorner", Float64) + yl, yl_line = _get_asc_field(lines, "yllcorner", Float64) + cs, cs_line = _get_asc_field(lines, "cellsize", Float64) + nodata, nodata_line = _get_asc_field(lines, "nodata", datatype) + # Read the data + M = zeros(datatype, (ncols, nrows)) + data_start = nodata_line+1 + data_end = length(lines) + for line_id in data_start:data_end + M[:,nrows-(line_id-(data_start))] = parse.(datatype, split(lines[line_id])) + end + println(permutedims(M)) + # Put data in the grid + grid = convert(Matrix{Union{datatype,Nothing}}, permutedims(M)) + for i in eachindex(M) + if grid[i] == nodata + grid[i] = nothing + end + end + @info ncols, nrows, cs + return SimpleSDMPredictor(grid, xl, xl+cs*2ncols, yl, yl+cs*2nrows) +end + +""" + ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} + +Writes a `layer` to a grid file, with a given `nodata` value. The layer must store numbers. +""" +function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} + if !(stride(layer)[1] ≈ stride(layer)[2]) + throw(DimensionMismatch("The cells of the layer to write must be square (i.e. both values of stride must be equal)")) + end + open(file, "w") do io + + write(io, "ncols $(size(layer, 2))\n") + write(io, "nrows $(size(layer, 1))\n") + write(io, "cellsize $(stride(layer)[1])\n") + write(io, "xllcorner $(layer.left)\n") + write(io, "yllcorner $(layer.bottom)\n") + write(io, "yllcorner $(layer.bottom)\n") + write(io, "nodata_value $(nodata)\n") + for row in reverse(1:size(layer.grid, 1)) + for el in layer.grid[row,:] + if isnothing(el) + write(io, "$(nodata) ") + else + write(io, "$(el) ") + end + end + write(io, "\n") + end + end +end diff --git a/test/ascii.jl b/test/ascii.jl new file mode 100644 index 00000000..ef057d8e --- /dev/null +++ b/test/ascii.jl @@ -0,0 +1,24 @@ +module SSLTestASCII +using SimpleSDMLayers +using Test + +M = rand(Float64, 4, 8) +M = convert(Matrix{Union{Nothing,Float64}}, M) +M[rand(Bool, size(M))] .= nothing +S = SimpleSDMPredictor(M, 0.0, 2.0, 0.0, 1.0) + +ascii(S, "test.asc") +U = ascii("test.asc") + +@test isfile("test.asc") +@test all(S.grid .== U.grid) +@test S.left == U.left +@test S.bottom == U.bottom +@test S.right == U.right +@test S.top == U.top +@test s +@test size(S) == size(U) + +rm("test.asc") + +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 018473c8..64f4c20e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,6 +9,7 @@ tests = [ "overloads" => "overloads.jl", "generated" => "generated.jl", "import" => "dataread.jl", + "ascii" => "ascii.jl", "worldclim" => "worldclim.jl", "landcover" => "landcover.jl", "chelsa" => "chelsa.jl", From 3c8a53ca4a28c5e428fd494ec877d15cb1ff96dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 09:25:46 -0500 Subject: [PATCH 02/13] =?UTF-8?q?=F0=9F=93=A6=20export=20ascii?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/SimpleSDMLayers.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/SimpleSDMLayers.jl b/src/SimpleSDMLayers.jl index a65298c6..ce5edea7 100644 --- a/src/SimpleSDMLayers.jl +++ b/src/SimpleSDMLayers.jl @@ -25,9 +25,9 @@ include(joinpath("datasets", "raster.jl")) include(joinpath("datasets", "worldclim.jl")) include(joinpath("datasets", "chelsa.jl")) include(joinpath("datasets", "landcover.jl")) -export worldclim -export bioclim -export landcover +include(joinpath("datasets", "ascii.jl")) +export worldclim, bioclim, landcover +export ascii include(joinpath("operations", "coarsen.jl")) include(joinpath("operations", "sliding.jl")) From c56e11e1c3aa5636e5b33b4b2ab5fa412d3c66d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 09:29:25 -0500 Subject: [PATCH 03/13] =?UTF-8?q?=F0=9F=91=90=20ascii=20returns=20the=20fi?= =?UTF-8?q?le=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index 7deab90c..155bf277 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -66,4 +66,9 @@ function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=con write(io, "\n") end end + return file +end + +function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} + return ascii(convert(SimpleSDMResponse, layer), file; nodata=nodata) end From e46b137ee0e37f590c794d75219516e106664d2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 09:35:53 -0500 Subject: [PATCH 04/13] =?UTF-8?q?=F0=9F=90=9B=20ascii=20is=20an=20overload?= =?UTF-8?q?=20of=20base?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index 155bf277..5c346624 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -9,7 +9,7 @@ end Reads the content of a grid file to a `SimpleSDMPredictor`, the type of which is given by the `datatype` argument. """ -function ascii(file::AbstractString, datatype::Type{T}=Float64) where {T <: Number} +function Base.ascii(file::AbstractString, datatype::Type{T}=Float64) where {T <: Number} lines = lowercase.(readlines(file)) # Get the information ncols, ncols_line = _get_asc_field(lines, "ncols", Int64) @@ -42,7 +42,7 @@ end Writes a `layer` to a grid file, with a given `nodata` value. The layer must store numbers. """ -function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} +function Base.ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} if !(stride(layer)[1] ≈ stride(layer)[2]) throw(DimensionMismatch("The cells of the layer to write must be square (i.e. both values of stride must be equal)")) end From eb45c549c7b2eda38bba64d0b7a20c24c38cbb5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 09:48:41 -0500 Subject: [PATCH 05/13] =?UTF-8?q?=F0=9F=91=90=20ascii=20not=20exported?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/src/man/data.md | 2 +- src/SimpleSDMLayers.jl | 1 - src/datasets/ascii.jl | 4 ++-- test/ascii.jl | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/src/man/data.md b/docs/src/man/data.md index f4b95141..1998f802 100644 --- a/docs/src/man/data.md +++ b/docs/src/man/data.md @@ -25,5 +25,5 @@ landcover ## ASCII files ```@docs -ascii +SimpleSDMLayers.ascii ``` \ No newline at end of file diff --git a/src/SimpleSDMLayers.jl b/src/SimpleSDMLayers.jl index ce5edea7..3197d0ec 100644 --- a/src/SimpleSDMLayers.jl +++ b/src/SimpleSDMLayers.jl @@ -27,7 +27,6 @@ include(joinpath("datasets", "chelsa.jl")) include(joinpath("datasets", "landcover.jl")) include(joinpath("datasets", "ascii.jl")) export worldclim, bioclim, landcover -export ascii include(joinpath("operations", "coarsen.jl")) include(joinpath("operations", "sliding.jl")) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index 5c346624..155bf277 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -9,7 +9,7 @@ end Reads the content of a grid file to a `SimpleSDMPredictor`, the type of which is given by the `datatype` argument. """ -function Base.ascii(file::AbstractString, datatype::Type{T}=Float64) where {T <: Number} +function ascii(file::AbstractString, datatype::Type{T}=Float64) where {T <: Number} lines = lowercase.(readlines(file)) # Get the information ncols, ncols_line = _get_asc_field(lines, "ncols", Int64) @@ -42,7 +42,7 @@ end Writes a `layer` to a grid file, with a given `nodata` value. The layer must store numbers. """ -function Base.ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} +function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} if !(stride(layer)[1] ≈ stride(layer)[2]) throw(DimensionMismatch("The cells of the layer to write must be square (i.e. both values of stride must be equal)")) end diff --git a/test/ascii.jl b/test/ascii.jl index ef057d8e..4333bdad 100644 --- a/test/ascii.jl +++ b/test/ascii.jl @@ -7,8 +7,8 @@ M = convert(Matrix{Union{Nothing,Float64}}, M) M[rand(Bool, size(M))] .= nothing S = SimpleSDMPredictor(M, 0.0, 2.0, 0.0, 1.0) -ascii(S, "test.asc") -U = ascii("test.asc") +SimpleSDMLayers.ascii(S, "test.asc") +U = SimpleSDMLayers.ascii("test.asc") @test isfile("test.asc") @test all(S.grid .== U.grid) From 667bdf5f921df9fcc08ba1ef5a5a5966b139e796 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 09:52:56 -0500 Subject: [PATCH 06/13] =?UTF-8?q?=F0=9F=90=9B=20fix=20dispatch=20for=20asc?= =?UTF-8?q?ii?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index 155bf277..ca891dd8 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -69,6 +69,6 @@ function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=con return file end -function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} - return ascii(convert(SimpleSDMResponse, layer), file; nodata=nodata) +function ascii(layer::SimpleSDMResponse{T}, file::AbstractString; nodata::T=convert(T, -9999)) where {T <: Number} + return ascii(convert(SimpleSDMPredictor, layer), file; nodata=nodata) end From c1ae466a6207f5a389a355ee48fe581fec39c411 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 09:53:50 -0500 Subject: [PATCH 07/13] =?UTF-8?q?=F0=9F=9A=A2=20new=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b7862520..b2ba8bb5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "SimpleSDMLayers" uuid = "2c645270-77db-11e9-22c3-0f302a89c64c" authors = ["Timothée Poisot ", "Gabriel Dansereau "] -version = "0.3.6" +version = "0.4.0" [deps] ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" From 7820f12e2df86680d30702f63ac0c36a5c0742b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 10:03:22 -0500 Subject: [PATCH 08/13] =?UTF-8?q?=F0=9F=94=A5=20revert=20to=20two=20argume?= =?UTF-8?q?nts=20startswith?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index ca891dd8..b5582ab4 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -1,5 +1,5 @@ function _get_asc_field(lines, field, type) - field_line = first(filter(startswith(field), lines)) + field_line = first(filter(line -> startswith(field, line), lines)) return parse(type, last(split(field_line, " "))), findfirst(startswith(field), lines) end From 1a213b19f6a598ba2be2cfe66c5e90be048279fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 13:12:07 -0500 Subject: [PATCH 09/13] =?UTF-8?q?=F0=9F=94=A7=20ignore=20test=20asc=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + test/ascii.jl | 4 +++- test/runtests.jl | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 2cb3b66e..e1552afc 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ assets test/assets test/gallery +test/*.asc # Files generated by invoking Julia with --track-allocation *.jl.mem diff --git a/test/ascii.jl b/test/ascii.jl index 4333bdad..18609b7c 100644 --- a/test/ascii.jl +++ b/test/ascii.jl @@ -5,12 +5,14 @@ using Test M = rand(Float64, 4, 8) M = convert(Matrix{Union{Nothing,Float64}}, M) M[rand(Bool, size(M))] .= nothing + S = SimpleSDMPredictor(M, 0.0, 2.0, 0.0, 1.0) SimpleSDMLayers.ascii(S, "test.asc") +@test isfile("test.asc") + U = SimpleSDMLayers.ascii("test.asc") -@test isfile("test.asc") @test all(S.grid .== U.grid) @test S.left == U.left @test S.bottom == U.bottom diff --git a/test/runtests.jl b/test/runtests.jl index 64f4c20e..66aae53d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,11 +5,11 @@ global anyerrors = false tests = [ "construction" => "construction.jl", + "ascii" => "ascii.jl", "basics" => "basics.jl", "overloads" => "overloads.jl", "generated" => "generated.jl", "import" => "dataread.jl", - "ascii" => "ascii.jl", "worldclim" => "worldclim.jl", "landcover" => "landcover.jl", "chelsa" => "chelsa.jl", From fa65bb23a501070c5210701938e7601001af4fba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 13:19:23 -0500 Subject: [PATCH 10/13] =?UTF-8?q?=F0=9F=90=9B=20fix=20order=20of=20argumen?= =?UTF-8?q?ts=20in=20startswith?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 3 +-- test/ascii.jl | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index b5582ab4..5b8aedd5 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -1,5 +1,5 @@ function _get_asc_field(lines, field, type) - field_line = first(filter(line -> startswith(field, line), lines)) + field_line = first(filter(line -> startswith(line, field), lines)) return parse(type, last(split(field_line, " "))), findfirst(startswith(field), lines) end @@ -47,7 +47,6 @@ function ascii(layer::SimpleSDMPredictor{T}, file::AbstractString; nodata::T=con throw(DimensionMismatch("The cells of the layer to write must be square (i.e. both values of stride must be equal)")) end open(file, "w") do io - write(io, "ncols $(size(layer, 2))\n") write(io, "nrows $(size(layer, 1))\n") write(io, "cellsize $(stride(layer)[1])\n") diff --git a/test/ascii.jl b/test/ascii.jl index 18609b7c..2442bbff 100644 --- a/test/ascii.jl +++ b/test/ascii.jl @@ -18,7 +18,6 @@ U = SimpleSDMLayers.ascii("test.asc") @test S.bottom == U.bottom @test S.right == U.right @test S.top == U.top -@test s @test size(S) == size(U) rm("test.asc") From c29350d35264afc8ac30068ae7811100cb9945a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 13:20:03 -0500 Subject: [PATCH 11/13] =?UTF-8?q?=F0=9F=94=A7=20remove=20the=20@info=20whe?= =?UTF-8?q?n=20reading=20ascii=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index 5b8aedd5..2d549831 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -33,7 +33,6 @@ function ascii(file::AbstractString, datatype::Type{T}=Float64) where {T <: Numb grid[i] = nothing end end - @info ncols, nrows, cs return SimpleSDMPredictor(grid, xl, xl+cs*2ncols, yl, yl+cs*2nrows) end From d208fc7ecc3b45f3b6502a589324e39f3d6b6930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 13:24:53 -0500 Subject: [PATCH 12/13] =?UTF-8?q?=F0=9F=90=9B=20startswith=20at=20it=20aga?= =?UTF-8?q?in?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index 2d549831..a600a9ea 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -1,6 +1,6 @@ function _get_asc_field(lines, field, type) field_line = first(filter(line -> startswith(line, field), lines)) - return parse(type, last(split(field_line, " "))), findfirst(startswith(field), lines) + return parse(type, last(split(field_line, " "))), findfirst(line -> startswith(line, field), lines) end """ From 7b55456a32f0c8c52cb1e29b80ceb2e8a7398829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Poisot?= Date: Sat, 20 Feb 2021 13:28:00 -0500 Subject: [PATCH 13/13] =?UTF-8?q?=E2=9A=A1=20single=20find=20operation=20i?= =?UTF-8?q?n=20=5Fget=5Fasc=5Ffield?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/datasets/ascii.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datasets/ascii.jl b/src/datasets/ascii.jl index a600a9ea..05e3f870 100644 --- a/src/datasets/ascii.jl +++ b/src/datasets/ascii.jl @@ -1,6 +1,6 @@ function _get_asc_field(lines, field, type) - field_line = first(filter(line -> startswith(line, field), lines)) - return parse(type, last(split(field_line, " "))), findfirst(line -> startswith(line, field), lines) + line_id = findfirst(line -> startswith(line, field), lines) + return parse(type, last(split(lines[line_id], " "))), line_id end """