From a102210243b5ce38e3b2d98344073b245cbce14a Mon Sep 17 00:00:00 2001 From: Nicholas Dinsmore Date: Fri, 15 Mar 2019 13:46:13 -0400 Subject: [PATCH 1/2] WIP better map value for abstract Dict --- base/abstractdict.jl | 86 +++++++++++++++++++++++++++++++++++++++++++- base/dict.jl | 35 +++++++++++++----- base/weakkeydict.jl | 16 ++++++++- 3 files changed, 126 insertions(+), 11 deletions(-) diff --git a/base/abstractdict.jl b/base/abstractdict.jl index e0dc4d08c8c87..11a42aaf7788f 100644 --- a/base/abstractdict.jl +++ b/base/abstractdict.jl @@ -702,6 +702,19 @@ function iterate(s::IdSet, state...) return (k, i) end + +""" + DirectStyle(::AbstractDict) +This method should return a value of type <: DirectIteratorStyle. +Currently the suppored styles are: + `NaiveDict` (default) which signifies to use the naive in place method which requires a hash evaulation + `SlottedDict` which is for `Dict` like `Dict` + `PointerDict` which returns an iterator function the provides a 0-dimmension array for access +Please see the trait definition for requires function +""" +DirectStyle(::Type{<:AbstractDict}) = NaiveDict() + + """ map!(f, values(dict::AbstractDict)) @@ -722,7 +735,8 @@ Dict{Symbol,Int64} with 2 entries: :b => 1 ``` """ -function map!(f, iter::ValueIterator) +map!( f, iter::Base.ValueIterator{D}) where D<:AbstractDict = _map!( DirectStyle(D), f, iter) +function _map!( ::NaiveDict, f, iter::Base.ValueIterator) # This is the naive fallback which requires hash evaluations # Contrary to the example Dict has an implementation which does not require hash evaluations dict = iter.dict @@ -731,3 +745,73 @@ function map!(f, iter::ValueIterator) end return iter end + + +""" + SlottedDict() +This trait signifies that the AbstractDict is structured similar to `Dict` +This means that it should have a LinearIndexable array of slots and an array of value. +Then be able to provide a function to provide a function which indicated whether a slot is filled. + Required Methods: slot_access_functions(::Type{<:AbstractDict}) +""" +struct SlottedDict <: DirectIteratorStyle end +# Dicts that return SlottedDicts must have the following methods +""" + slot_access_functions(::Type{CustomDict}) +This function should return a tuple of three function that will be used to access values + ( isslotfilled(slots::AbstractArray, index)::Bool, + get_slots_array(dict::CustomDict)::AbstractArray, + get_value_array(dict::CustomDict)::AbstractArray) +Importantly these functions do not have to be named as such they just need to take the arguments as shown +It is important that the arrays returned are the same length +""" +function slot_access_functions(::Type{<:AbstractDict}) end + + +function _map!(::SlottedDict, f, iter::Base.ValueIterator{D}) where D<:AbstractDict + (isslotfilled, get_slots, get_vals) = slot_access_functions(D) + slots = get_slots(iter.dict) + vals = get_vals(iter.dict) + # @inbounds is here so the it gets propigated to isslotfiled + @inbounds for i = 1:length(slots) + if isslotfilled(slots, i) + vals[i] = f(vals[i]) + end + end + return iter +end + +""" + PointerDict() +This trait signifies that the AbstractDict has an iterate like function which on each +iteration provides an `AbstractArray{T,N}` which acts as a pointer to that given value + + Required Methods: `iterate_value_pointer(d::Abstract{K,V})` + `iterate_value_pointer(d::Abstract{K,V}, state)` + Both of which should act like iterate returning either: + `Nothing` to signify the iteration is over + `(value_pointer::AbstractArray{V,0},state)` +""" +struct PointerDict <: DirectIteratorStyle end + +""" + `iterate_value_pointer(d::Abstract{K,V})` + `iterate_value_pointer(d::Abstract{K,V}, state)` + Both of which should act like iterate returning either: + `Nothing` to signify the iteration is over + `(value_pointer::AbstractArray{V,0},state)` +Importantly these functions Do not have to be named as such they just need to take the arguments as shown +It is important that the arrays returned are the same length +""" + +function _map!(::PointerDict, f, iter::Base.ValueIterator{D}) where D<:AbstractDict + dict=iter.dict + @inbounds next = iterate_value_pointer(dict) + while next !== nothing + (v, state) = next + @inbounds v[] = f(v[]) + @inbounds next = iterate_value_pointer(dict, state) + end + return iter +end +#--------------------- diff --git a/base/dict.jl b/base/dict.jl index e6ef7f312a578..c51c7d9c706cf 100644 --- a/base/dict.jl +++ b/base/dict.jl @@ -686,16 +686,33 @@ end filter!(f, d::Dict) = filter_in_one_pass!(f, d) -function map!(f, iter::ValueIterator{<:Dict}) - dict = iter.dict - vals = dict.vals - # @inbounds is here so the it gets propigated to isslotfiled - @inbounds for i = dict.idxfloor:lastindex(vals) - if isslotfilled(dict, i) - vals[i] = f(vals[i]) - end +# function map!(f, iter::ValueIterator{<:Dict}) +# dict = iter.dict +# vals = dict.vals +# # @inbounds is here so the it gets propigated to isslotfiled +# @inbounds for i = dict.idxfloor:lastindex(vals) +# if isslotfilled(dict, i) +# vals[i] = f(vals[i]) +# end +# end +# return iter +# end + +DirectStyle(::Type{<:Dict}) = SlottedDict() +@inline function slot_access_functions(::Type{<:Dict}) + return ( ((slots, i) -> @inbounds slots[i] == 0x1), #This is the slot test function + dict->Base.unsafe_view(dict.slots, dict.idxfloor:lastindex(dict.vals)), # This is the function that gets the slots array + dict->Base.unsafe_view(dict.vals, dict.idxfloor:lastindex(dict.vals)) ) # This is the function that gets the value array +end + +# This is included as an example +function iterate_value_pointer(d::Dict,s::Int=d.idxfloor) + L = Base.lastindex(d.slots) + @inbounds while s <= lastindex(d.slots) && !Base.isslotfilled(d,s) + s += 1 end - return iter + s > L && return nothing + return (@inbounds Base.unsafe_view(d.vals,s), s+1) end struct ImmutableDict{K,V} <: AbstractDict{K,V} diff --git a/base/weakkeydict.jl b/base/weakkeydict.jl index db1b60e27d771..9dbe1e1ce0436 100644 --- a/base/weakkeydict.jl +++ b/base/weakkeydict.jl @@ -92,7 +92,20 @@ function getkey(wkh::WeakKeyDict{K}, kk, default) where K end end -map!(f,iter::ValueIterator{<:WeakKeyDict})= map!(f, values(iter.dict.ht)) +# map!(f,iter::ValueIterator{<:WeakKeyDict})= map!(f, values(iter.dict.ht)) +# WeakKeyDict could easily be of trait SlottedDict but it is this way for example. +DirectStyle(::Type{<:WeakKeyDict}) = PointerDict() +function iterate_value_pointer(wd::WeakKeyDict,s::Int=wd.ht.idxfloor) + d=wd.ht + L = Base.lastindex(d.slots) + @inbounds while s <= lastindex(d.slots) && !Base.isslotfilled(d,s) + s += 1 + end + s > L && return nothing + return (@inbounds Base.unsafe_view(d.vals,s), s+1) +end + + get(wkh::WeakKeyDict{K}, key, default) where {K} = lock(() -> get(wkh.ht, key, default), wkh) get(default::Callable, wkh::WeakKeyDict{K}, key) where {K} = lock(() -> get(default, wkh.ht, key), wkh) function get!(wkh::WeakKeyDict{K}, key, default) where {K} @@ -112,6 +125,7 @@ getindex(wkh::WeakKeyDict{K}, key) where {K} = lock(() -> getindex(wkh.ht, key), isempty(wkh::WeakKeyDict) = isempty(wkh.ht) length(t::WeakKeyDict) = length(t.ht) + function iterate(t::WeakKeyDict{K,V}) where V where K gc_token = Ref{Bool}(false) # no keys will be deleted via finalizers until this token is gc'd finalizer(gc_token) do r From 4e871bb63b77c6c82492857a10b53899ecb75c55 Mon Sep 17 00:00:00 2001 From: Nicholas Dinsmore Date: Fri, 15 Mar 2019 14:17:57 -0400 Subject: [PATCH 2/2] small fix --- base/abstractdict.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base/abstractdict.jl b/base/abstractdict.jl index 11a42aaf7788f..d341fb4e32401 100644 --- a/base/abstractdict.jl +++ b/base/abstractdict.jl @@ -702,6 +702,9 @@ function iterate(s::IdSet, state...) return (k, i) end +abstract type DirectIteratorStyle end +# This will be the fall back that uses the naive map above +struct NaiveDict <: DirectIteratorStyle end """ DirectStyle(::AbstractDict)