diff --git a/README.md b/README.md index 46768f9a..aa6fd6e7 100644 --- a/README.md +++ b/README.md @@ -211,3 +211,40 @@ end s = [MyType(rand(), a=1, b=2) for i in 1:10] StructArray(s) ``` + +## Advanced: mutate-or-widen style accumulation + +StructArrays provides a function `StructArrays.append!!(dest, src)` (unexported) for "mutate-or-widen" style accumulation. This function can be used via [`BangBang.append!!`](https://tkf.github.io/BangBang.jl/dev/#BangBang.append!!-Tuple{Any,Any}) and [`BangBang.push!!`](https://tkf.github.io/BangBang.jl/dev/#BangBang.push!!-Tuple{Any,Any,Any,Vararg{Any,N}%20where%20N}) as well. + +`StructArrays.append!!` works like `append!(dest, src)` if `dest` can contain all element types in `src` iterator; i.e., it _mutates_ `dest` in-place: + +```julia +julia> dest = StructVector((a=[1], b=[2])) +1-element StructArray(::Array{Int64,1}, ::Array{Int64,1}) with eltype NamedTuple{(:a, :b),Tuple{Int64,Int64}}: + (a = 1, b = 2) + +julia> StructArrays.append!!(dest, [(a = 3, b = 4)]) +2-element StructArray(::Array{Int64,1}, ::Array{Int64,1}) with eltype NamedTuple{(:a, :b),Tuple{Int64,Int64}}: + (a = 1, b = 2) + (a = 3, b = 4) + +julia> ans === dest +true +``` + +Unlike `append!`, `append!!` can also _widen_ element type of `dest` array element types: + +```julia +julia> StructArrays.append!!(dest, [(a = missing, b = 6)]) +3-element StructArray(::Array{Union{Missing, Int64},1}, ::Array{Int64,1}) with eltype NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}: + NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}((1, 2)) + NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}((3, 4)) + NamedTuple{(:a, :b),Tuple{Union{Missing, Int64},Int64}}((missing, 6)) + +julia> ans === dest +false +``` + +Since the original array `dest` cannot hold the input, a new array is created (`ans !== dest`). + +Combined with [function barriers](https://docs.julialang.org/en/latest/manual/performance-tips/#kernel-functions-1), `append!!` is a useful building block for implementing `collect`-like functions. diff --git a/src/collect.jl b/src/collect.jl index 21979895..bceb1a68 100644 --- a/src/collect.jl +++ b/src/collect.jl @@ -128,3 +128,41 @@ function widenarray(dest::AbstractArray, i, ::Type{T}) where T copyto!(new, 1, dest, 1, i-1) new end + +""" +`append!!(dest, itr) -> dest′` + +Try to append `itr` into a vector `dest`. Widen element type of +`dest` if it cannot hold the elements of `itr`. That is to say, + +```julia +vcat(dest, StructVector(itr)) == append!!(dest, itr) +``` + +holds. Note that `dest′` may or may not be the same object as `dest`. +The state of `dest` is unpredictable after `append!!` +is called (e.g., it may contain just half of the elements from `itr`). +""" +append!!(dest::AbstractVector, itr) = + _append!!(dest, itr, Base.IteratorSize(itr)) + +function _append!!(dest::AbstractVector, itr, ::Union{Base.HasShape, Base.HasLength}) + n = length(itr) # itr may be stateful so do this first + fr = iterate(itr) + fr === nothing && return dest + el, st = fr + i = lastindex(dest) + 1 + if iscompatible(el, dest) + resize!(dest, length(dest) + n) + @inbounds dest[i] = el + return collect_to_structarray!(dest, itr, i + 1, st) + else + new = widenstructarray(dest, i, el) + resize!(new, length(dest) + n) + @inbounds new[i] = el + return collect_to_structarray!(new, itr, i + 1, st) + end +end + +_append!!(dest::AbstractVector, itr, ::Base.SizeUnknown) = + grow_to_structarray!(dest, itr) diff --git a/test/runtests.jl b/test/runtests.jl index f699f8b1..03d76c45 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,5 @@ using StructArrays -using StructArrays: staticschema, iscompatible, _promote_typejoin +using StructArrays: staticschema, iscompatible, _promote_typejoin, append!! using OffsetArrays: OffsetArray import Tables, PooledArrays, WeakRefStrings using Test @@ -646,3 +646,24 @@ end str = String(take!(io)) @test str == "StructArray(::Array{Int64,1}, ::Array{Int64,1})" end + +@testset "append!!" begin + dest_examples = [ + ("mutate", StructVector(a = [1], b = [2])), + ("widen", StructVector(a = [1], b = [nothing])), + ] + itr = [(a = 1, b = 2), (a = 1, b = 2), (a = 1, b = 12)] + itr_examples = [ + ("HasLength", () -> itr), + ("SizeUnknown", () -> (x for x in itr if isodd(x.a))), + # Broken due to https://github.com/JuliaArrays/StructArrays.jl/issues/100: + # ("empty", (x for x in itr if false)), + # Broken due to https://github.com/JuliaArrays/StructArrays.jl/issues/99: + # ("stateful", () -> Iterators.Stateful(itr)), + ] + @testset "$destlabel $itrlabel" for (destlabel, dest) in dest_examples, + (itrlabel, makeitr) in itr_examples + + @test vcat(dest, StructVector(makeitr())) == append!!(copy(dest), makeitr()) + end +end