Revised printing & improved softmax bias initialization

Project.toml

@@ -1,7 +1,7 @@
 name = "EvoTrees"
 uuid = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5"
 authors = ["jeremiedb <[email protected]>"]
-version = "0.14.4"
+version = "0.14.5"
 
 [deps]
 BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"

experiments/benchmarks-regressor.jl

@@ -43,14 +43,14 @@ params_xgb = Dict(
     :max_bin => 64,
 )
 
-# dtrain = DMatrix(x_train, y_train .- 1)
-# watchlist = Dict("train" => DMatrix(x_train, y_train .- 1))
+dtrain = DMatrix(x_train, y_train .- 1)
+watchlist = Dict("train" => DMatrix(x_train, y_train .- 1))
+@time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...);
 # @time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...);
-# # @time m_xgb = xgboost(dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...);
-# @btime m_xgb = xgboost($dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...);
-# @info "xgboost predict:"
-# @time pred_xgb = XGBoost.predict(m_xgb, x_train);
-# @btime XGBoost.predict($m_xgb, $x_train);
+@btime m_xgb = xgboost($dtrain; watchlist, nthread=nthread, verbosity=0, eval_metric = metric_xgb, params_xgb...);
+@info "xgboost predict:"
+@time pred_xgb = XGBoost.predict(m_xgb, x_train);
+@btime XGBoost.predict($m_xgb, $x_train);
 
 # @info "lightgbm train:"
 # m_gbm = LGBMRegression(
@@ -103,14 +103,15 @@ params_evo = EvoTreeRegressor(
     nbins=64,
     rng = 123,
 )
-# params_evo.device = "cpu"
+
+params_evo.device = "cpu"
+@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
 # @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
-# # @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
-# # @time m_evo = fit_evotree(params_evo; x_train, y_train);
-# @btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
-# @info "evotrees predict CPU:"
-# @time pred_evo = EvoTrees.predict(m_evo, x_train);
-# @btime EvoTrees.predict($m_evo, $x_train);
+# @time m_evo = fit_evotree(params_evo; x_train, y_train);
+@btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
+@info "evotrees predict CPU:"
+@time pred_evo = EvoTrees.predict(m_evo, x_train);
+@btime EvoTrees.predict($m_evo, $x_train);
 
 @info "evotrees train GPU:"
 params_evo.device = "gpu"

src/EvoTrees.jl

@@ -12,7 +12,7 @@ export EvoTreeRegressor,
 
 using Base.Threads: @threads, @spawn, nthreads, threadid
 using Statistics
-using StatsBase: sample, sample!, quantile
+using StatsBase: sample, sample!, quantile, proportions
 using Random
 using Random: seed!, AbstractRNG
 using Distributions

src/fit.jl

@@ -35,7 +35,8 @@ function init_evotree(
             y = UInt32.(CategoricalArrays.levelcode.(yc))
         end
         K = length(levels)
-        μ = zeros(T, K)
+        μ = T.(log.(proportions(y)))
+        μ .-= maximum(μ)
         !isnothing(offset) && (offset .= log.(offset))
     elseif L == GaussianMLE
         K = 2
@@ -322,6 +323,8 @@ function fit_evotree(
     return_logger=false
 ) where {L,T}
 
+    verbosity == 1 && @info params
+
     # initialize model and cache
     if String(params.device) == "gpu"
         m, cache = init_evotree_gpu(params; x_train, y_train, w_train, offset_train, fnames)

src/gpu/fit_gpu.jl

@@ -2,9 +2,9 @@ function init_evotree_gpu(
     params::EvoTypes{L,T};
     x_train::AbstractMatrix,
     y_train::AbstractVector,
-    w_train=nothing,
-    offset_train=nothing,
-    fnames=nothing
+    w_train = nothing,
+    offset_train = nothing,
+    fnames = nothing,
 ) where {L,T}
 
     levels = nothing
@@ -27,11 +27,12 @@ function init_evotree_gpu(
             y = CuArray(UInt32.(CategoricalArrays.levelcode.(y_train)))
         else
             levels = sort(unique(y_train))
-            yc = CategoricalVector(y_train, levels=levels)
+            yc = CategoricalVector(y_train, levels = levels)
             y = CuArray(UInt32.(CategoricalArrays.levelcode.(yc)))
         end
         K = length(levels)
-        μ = zeros(T, K)
+        μ = T.(log.(proportions(y)))
+        μ .-= maximum(μ)
         !isnothing(offset) && (offset .= log.(offset))
     elseif L == GaussianMLE
         K = 2
@@ -76,7 +77,10 @@ function init_evotree_gpu(
     js = zeros(eltype(js_), ceil(Int, params.colsample * x_size[2]))
 
     # initialize histograms
-    nodes = [TrainNodeGPU(x_size[2], params.nbins, K, view(is_in, 1:0), T) for n = 1:2^params.max_depth-1]
+    nodes = [
+        TrainNodeGPU(x_size[2], params.nbins, K, view(is_in, 1:0), T) for
+        n = 1:2^params.max_depth-1
+    ]
     out = CUDA.zeros(UInt32, x_size[1])
     left = CUDA.zeros(UInt32, x_size[1])
     right = CUDA.zeros(UInt32, x_size[1])
@@ -89,23 +93,23 @@ function init_evotree_gpu(
 
     # store cache
     cache = (
-        info=Dict(:nrounds => 0),
-        x=CuArray(x),
-        x_bin=x_bin,
-        y=y,
-        nodes=nodes,
-        pred=pred,
-        is_in=is_in,
-        is_out=is_out,
-        mask=mask,
-        js_=js_,
-        js=js,
-        out=out,
-        left=left,
-        right=right,
-        ∇=∇,
-        edges=edges,
-        monotone_constraints=CuArray(monotone_constraints),
+        info = Dict(:nrounds => 0),
+        x = CuArray(x),
+        x_bin = x_bin,
+        y = y,
+        nodes = nodes,
+        pred = pred,
+        is_in = is_in,
+        is_out = is_out,
+        mask = mask,
+        js_ = js_,
+        js = js,
+        out = out,
+        left = left,
+        right = right,
+        ∇ = ∇,
+        edges = edges,
+        monotone_constraints = CuArray(monotone_constraints),
     )
 
     return m, cache
@@ -121,9 +125,10 @@ function grow_evotree!(
     # compute gradients
     update_grads_gpu!(cache.∇, cache.pred, cache.y, params)
     # subsample rows
-    cache.nodes[1].is = subsample_gpu(cache.is_in, cache.is_out, cache.mask, params.rowsample, params.rng)
+    cache.nodes[1].is =
+        subsample_gpu(cache.is_in, cache.is_out, cache.mask, params.rowsample, params.rng)
     # subsample cols
-    sample!(params.rng, cache.js_, cache.js, replace=false, ordered=true)
+    sample!(params.rng, cache.js_, cache.js, replace = false, ordered = true)
 
     # assign a root and grow tree
     tree = TreeGPU{L,K,T}(params.max_depth)
@@ -174,7 +179,7 @@ function grow_tree_gpu!(
     end
 
     # initialize summary stats
-    nodes[1].∑ .= vec(sum(∇[:, nodes[1].is], dims=2))
+    nodes[1].∑ .= vec(sum(∇[:, nodes[1].is], dims = 2))
     nodes[1].gain = get_gain(params, Array(nodes[1].∑)) # should use a GPU version?
 
     # grow while there are remaining active nodes - TO DO histogram substraction hits issue on GPU

src/gpu/structs_gpu.jl

@@ -53,10 +53,24 @@ TreeGPU{L,K,T}(depth::Int) where {L,K,T} = TreeGPU{L,K,T}(
     CUDA.zeros(Bool, 2^depth - 1),
 )
 
+function Base.show(io::IO, tree::TreeGPU)
+    println(io, "$(typeof(tree))")
+    for fname in fieldnames(typeof(tree))
+        println(io, " - $fname: $(getfield(tree, fname))")
+    end
+end
+
 # gradient-boosted tree is formed by a vector of trees
 struct EvoTreeGPU{L,K,T}
     trees::Vector{TreeGPU{L,K,T}}
     info::Any
 end
 (m::EvoTreeGPU)(x::AbstractMatrix) = predict(m, x)
 get_types(::EvoTreeGPU{L,K,T}) where {L,K,T} = (L, T)
+
+function Base.show(io::IO, evotree::EvoTreeGPU)
+    println(io, "$(typeof(evotree))")
+    println(io, " - Contains $(length(evotree.trees)) trees in field `trees` (incl. 1 bias tree).")
+    println(io, " - Data input has $(length(evotree.info[:fnames])) features.")
+    println(io, " - $(keys(evotree.info)) info accessible in field `info`")
+end

src/models.jl

@@ -65,16 +65,6 @@ function EvoTreeRegressor(; kwargs...)
         :device => "cpu",
     )
 
-    args_ignored = setdiff(keys(kwargs), keys(args))
-    args_ignored_str = join(args_ignored, ", ")
-    length(args_ignored) > 0 &&
-        @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)."
-
-    args_default = setdiff(keys(args), keys(kwargs))
-    args_default_str = join(args_default, ", ")
-    length(args_default) > 0 &&
-        @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)."
-
     args_override = intersect(keys(args), keys(kwargs))
     for arg in args_override
         args[arg] = kwargs[arg]
@@ -158,16 +148,6 @@ function EvoTreeCount(; kwargs...)
         :device => "cpu",
     )
 
-    args_ignored = setdiff(keys(kwargs), keys(args))
-    args_ignored_str = join(args_ignored, ", ")
-    length(args_ignored) > 0 &&
-        @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)."
-
-    args_default = setdiff(keys(args), keys(kwargs))
-    args_default_str = join(args_default, ", ")
-    length(args_default) > 0 &&
-        @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)."
-
     args_override = intersect(keys(args), keys(kwargs))
     for arg in args_override
         args[arg] = kwargs[arg]
@@ -230,16 +210,6 @@ function EvoTreeClassifier(; kwargs...)
         :device => "cpu",
     )
 
-    args_ignored = setdiff(keys(kwargs), keys(args))
-    args_ignored_str = join(args_ignored, ", ")
-    length(args_ignored) > 0 &&
-        @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)."
-
-    args_default = setdiff(keys(args), keys(kwargs))
-    args_default_str = join(args_default, ", ")
-    length(args_default) > 0 &&
-        @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)."
-
     args_override = intersect(keys(args), keys(kwargs))
     for arg in args_override
         args[arg] = kwargs[arg]
@@ -304,16 +274,6 @@ function EvoTreeMLE(; kwargs...)
         :device => "cpu",
     )
 
-    args_ignored = setdiff(keys(kwargs), keys(args))
-    args_ignored_str = join(args_ignored, ", ")
-    length(args_ignored) > 0 &&
-        @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)."
-
-    args_default = setdiff(keys(args), keys(kwargs))
-    args_default_str = join(args_default, ", ")
-    length(args_default) > 0 &&
-        @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)."
-
     args_override = intersect(keys(args), keys(kwargs))
     for arg in args_override
         args[arg] = kwargs[arg]
@@ -388,16 +348,6 @@ function EvoTreeGaussian(; kwargs...)
         :device => "cpu",
     )
 
-    args_ignored = setdiff(keys(kwargs), keys(args))
-    args_ignored_str = join(args_ignored, ", ")
-    length(args_ignored) > 0 &&
-        @info "Following $(length(args_ignored)) provided arguments will be ignored: $(args_ignored_str)."
-
-    args_default = setdiff(keys(args), keys(kwargs))
-    args_default_str = join(args_default, ", ")
-    length(args_default) > 0 &&
-        @info "Following $(length(args_default)) arguments were not provided and will be set to default: $(args_default_str)."
-
     args_override = intersect(keys(args), keys(kwargs))
     for arg in args_override
         args[arg] = kwargs[arg]
@@ -434,4 +384,11 @@ const EvoTypes{L,T} = Union{
     EvoTreeMLE{L,T},
 }
 
-get_types(::EvoTypes{L,T}) where {L,T} = (L, T)
+get_types(::EvoTypes{L,T}) where {L,T} = (L, T)
+
+function Base.show(io::IO, config::EvoTypes)
+    println(io, "$(typeof(config))")
+    for fname in fieldnames(typeof(config))
+        println(io, " - $fname: $(getfield(config, fname))")
+    end
+end

src/structs.jl

@@ -56,10 +56,24 @@ function Tree{L,K,T}(depth::Int) where {L,K,T}
     )
 end
 
+function Base.show(io::IO, tree::Tree)
+    println(io, "$(typeof(tree))")
+    for fname in fieldnames(typeof(tree))
+        println(io, " - $fname: $(getfield(tree, fname))")
+    end
+end
+
 # gradient-boosted tree is formed by a vector of trees
 struct EvoTree{L,K,T}
     trees::Vector{Tree{L,K,T}}
     info::Dict
 end
 (m::EvoTree)(x::AbstractMatrix) = predict(m, x)
 get_types(::EvoTree{L,K,T}) where {L,K,T} = (L,T)
+
+function Base.show(io::IO, evotree::EvoTree)
+    println(io, "$(typeof(evotree))")
+    println(io, " - Contains $(length(evotree.trees)) trees in field `trees` (incl. 1 bias tree).")
+    println(io, " - Data input has $(length(evotree.info[:fnames])) features.")
+    println(io, " - $(keys(evotree.info)) info accessible in field `info`")
+end

test/MLJ.jl

@@ -115,7 +115,7 @@ tree_model = EvoTreeClassifier(
 # @load EvoTreeRegressor
 mach = machine(tree_model, X, y)
 train, test = partition(eachindex(y), 0.7, shuffle = true); # 70:30 split
-fit!(mach, rows = train, verbosity = 1)
+fit!(mach, rows = train, verbosity = 1);
 
 mach.model.nrounds += 50
 fit!(mach, rows = train, verbosity = 1)