Merge pull request #188 from Evovest/gpu-upgrade

Gpu upgrade

experiments/MLJ.jl

@@ -58,18 +58,21 @@ println(mean(abs.(pred_test - selectrows(Y, test))))
 X, y_train = @load_crabs
 x_train = matrix(X)
 
+using CUDA
+CUDA.allowscalar(false)
+
 # define hyperparameters
 config = EvoTreeClassifier(
     max_depth = 4,
     eta = 0.05,
     lambda = 0.0,
     gamma = 0.0,
     nbins = 32,
-    nrounds = 100,
+    nrounds = 200,
     device = "cpu"
 )
 model = fit_evotree(config; x_train, y_train);
-model = fit_evotree(config; x_train, y_train, x_eval = x_train, y_eval = y_train, metric=:mlogloss, print_every_n=5);
+model = fit_evotree(config; x_train, y_train, x_eval = x_train, y_eval = y_train, metric=:mlogloss, print_every_n=10, early_stopping_rounds=25);
 
 pred = predict(model, x_train)
 pred_cat = pred .> 0.5

experiments/benchmarks-MLE.jl

@@ -72,8 +72,8 @@ y_train = rand(size(x_train, 1))
 
 @info "evotrees train CPU:"
 params_evo.device = "cpu"
-@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=:logistic, print_every_n=100);
-@btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=:logistic);
+@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=:logistic_mle, print_every_n=100);
+@btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=:logistic_mle);
 @info "evotrees predict CPU:"
 @time pred_evo = EvoTrees.predict(m_evo, x_train);
 @btime EvoTrees.predict($m_evo, $x_train);

experiments/benchmarks-regressor.jl

@@ -74,11 +74,11 @@ params_evo.device = "cpu"
 @time pred_evo = EvoTrees.predict(m_evo, x_train);
 @btime EvoTrees.predict($m_evo, $x_train);
 
-# @info "evotrees train GPU:"
-# params_evo.device = "gpu"
-# @time m_evo_gpu = fit_evotree(params_evo; x_train, y_train);
-# @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
-# @btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
-# @info "evotrees predict GPU:"
-# @time pred_evo = EvoTrees.predict(m_evo_gpu, x_train);
-# @btime EvoTrees.predict($m_evo_gpu, $x_train);
+@info "evotrees train GPU:"
+params_evo.device = "gpu"
+@time m_evo_gpu = fit_evotree(params_evo; x_train, y_train);
+@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
+@btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
+@info "evotrees predict GPU:"
+@time pred_evo = EvoTrees.predict(m_evo_gpu, x_train);
+@btime EvoTrees.predict($m_evo_gpu, $x_train);

experiments/benchmarks-softmax.jl

@@ -7,51 +7,43 @@ using BenchmarkTools
 using CUDA
 
 nrounds = 200
-num_class = 10
+num_class = 5
 nthread = Base.Threads.nthreads()
 
 @info nthread
-loss = "linear"
-if loss == "linear"
-    loss_xgb = "reg:squarederror"
-    metric_xgb = "mae"
-    loss_evo = :linear
-    metric_evo = :mae
-elseif loss == "logistic"
-    loss_xgb = "reg:logistic"
-    metric_xgb = "logloss"
-    loss_evo = :logistic
-    metric_evo = :logloss
-end
+loss_xgb = "multi:softmax"
+metric_xgb = "mlogloss"
+metric_evo = :mlogloss
+
 
 # xgboost aprams
-# params_xgb = [
-#     "max_depth" => 5,
-#     "eta" => 0.05,
-#     "objective" => loss_xgb,
-#     "print_every_n" => 5,
-#     "subsample" => 0.5,
-#     "colsample_bytree" => 0.5,
-#     "tree_method" => "hist",
-#     "max_bin" => 64,
-# ]
-# metrics = [metric_xgb]
+params_xgb = [
+    "max_depth" => 5,
+    "eta" => 0.05,
+    "objective" => loss_xgb,
+    "print_every_n" => 5,
+    "subsample" => 0.5,
+    "colsample_bytree" => 0.5,
+    "tree_method" => "hist",
+    "max_bin" => 64,
+    "num_class" => num_class,
+]
+metrics = [metric_xgb]
 
 # EvoTrees params
 params_evo = EvoTreeClassifier(
     T=Float32,
-    loss=loss_evo,
     nrounds=nrounds,
     alpha=0.5,
     lambda=0.0,
     gamma=0.0,
     eta=0.05,
     max_depth=6,
-    min_weight=1.0,
+    min_weight=10.0,
     rowsample=0.5,
     colsample=0.5,
     nbins=64,
-    num_class = num_class
+    num_class=num_class
 )
 
 nobs = Int(1e6)
@@ -60,27 +52,29 @@ num_feat = Int(100)
 x_train = rand(nobs, num_feat)
 y_train = rand(1:num_class, size(x_train, 1))
 
-# @info "xgboost train:"
-# @time m_xgb = xgboost(x_train, nrounds, label=y_train, param=params_xgb, metrics=metrics, nthread=nthread, silent=1);
-# @btime xgboost($x_train, $nrounds, label=$y_train, param=$params_xgb, metrics=$metrics, nthread=$nthread, silent=1);
-# @info "xgboost predict:"
-# @time pred_xgb = XGBoost.predict(m_xgb, x_train);
-# @btime XGBoost.predict($m_xgb, $x_train);
+@info "xgboost train:"
+@time m_xgb = xgboost(x_train, nrounds, label=y_train .- 1 , param=params_xgb, metrics=metrics, nthread=nthread, silent=1);
+@time m_xgb = xgboost(x_train, nrounds, label=y_train .- 1 , param=params_xgb, metrics=metrics, nthread=nthread, silent=1);
+# @btime xgboost($x_train, $nrounds, label=$y_train .- 1, param=$params_xgb, metrics=$metrics, nthread=$nthread, silent=1);
+@info "xgboost predict:"
+@time pred_xgb = XGBoost.predict(m_xgb, x_train);
+@btime XGBoost.predict($m_xgb, $x_train);
 
 @info "evotrees train CPU:"
 params_evo.device = "cpu"
 @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
-@btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
-@btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train);
+@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
+# @btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
+# @btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train);
 @info "evotrees predict CPU:"
 @time pred_evo = EvoTrees.predict(m_evo, x_train);
 @btime EvoTrees.predict($m_evo, $x_train);
 
 @info "evotrees train GPU:"
 params_evo.device = "gpu"
-@time m_evo_gpu = fit_evotree(params_evo; x_train, y_train);
 @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
-@btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
+@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, print_every_n=100);
+# @btime fit_evotree($params_evo; x_train=$x_train, y_train=$y_train, x_eval=$x_train, y_eval=$y_train, metric=metric_evo);
 @info "evotrees predict GPU:"
 @time pred_evo = EvoTrees.predict(m_evo_gpu, x_train);
 @btime EvoTrees.predict($m_evo_gpu, $x_train);

experiments/GPU_v2.jl → experiments/gpu-upgrade.jl

@@ -1,10 +1,25 @@
+using Revise
+using EvoTrees
 # using CUDA
 using CUDA
+using BenchmarkTools
 # using Flux
 # using GeometricFlux
 
-nbins = 32
-ncol = 100
+nobs = 500_000
+nbins = 64
+ncol = 50
+
+h∇ = CUDA.zeros(Float32, 3, nbins, ncol)
+∇ = CUDA.rand(Float32, 3, nobs)
+x_bin = CuArray(rand(UInt32.(1:nbins), nobs, ncol))
+𝑖 = CuArray(UInt32.(1:nobs))
+𝑗 = CuArray(UInt32.(1:ncol))
+
+h∇ .= 0
+CUDA.@time EvoTrees.update_hist_gpu!(h∇, ∇, x_bin, 𝑖, 𝑗)
+@btime EvoTrees.update_hist_gpu!($h∇, $∇, $x_bin, $𝑖, $𝑗)
+
 items = Int(1e6)
 hist = zeros(Float32, nbins, ncol)
 δ = rand(Float32, items)

experiments/readme_plots_cpu.jl

@@ -30,12 +30,12 @@ y_train, y_eval = Y[𝑖_train], Y[𝑖_eval]
 params1 = EvoTreeRegressor(T=Float32,
     loss=:linear,
     nrounds=200, nbins=64,
-    lambda=0.1, gamma=0.1, eta=0.05,
+    lambda=0.01, gamma=0.1, eta=0.05,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0,
     rng=123)
 
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, metric=:mse, print_every_n=25);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, metric=:mse, print_every_n=25, early_stopping_rounds=20);
 # 67.159 ms (77252 allocations: 28.06 MiB)
 # @time model = fit_evotree(params1, X_train, Y_train, X_eval = X_eval, Y_eval = Y_eval, print_every_n = 999);
 # @btime model = fit_evotree($params1, $X_train, $Y_train, X_eval = $X_eval, Y_eval = $Y_eval);
@@ -54,15 +54,15 @@ mean((pred_eval_linear .- y_eval) .^ 2)
 # linear weighted
 params1 = EvoTreeRegressor(T=Float64,
     loss=:linear,
-    nrounds=200, nbins=64,
+    nrounds=500, nbins=64,
     lambda=0.1, gamma=0.1, eta=0.05,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0,
     rng=123)
 
 # W_train = ones(eltype(Y_train), size(Y_train)) .* 5
 w_train = rand(eltype(y_train), size(y_train)) .+ 0
-@time model = fit_evotree(params1; x_train, y_train, w_train, x_eval, y_eval, print_every_n=25, metric=:mse);
+@time model = fit_evotree(params1; x_train, y_train, w_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:mse);
 # 67.159 ms (77252 allocations: 28.06 MiB)
 # @time model = fit_evotree(params1, X_train, Y_train, X_eval = X_eval, Y_eval = Y_eval, print_every_n = 999);
 # @btime model = fit_evotree($params1, $X_train, $Y_train, X_eval = $X_eval, Y_eval = $Y_eval);
@@ -84,7 +84,7 @@ params1 = EvoTreeRegressor(
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
 
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, metric=:logloss);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:logloss);
 # 218.040 ms (123372 allocations: 34.71 MiB)
 # @btime model = fit_evotree($params1, $X_train, $Y_train, X_eval = $X_eval, Y_eval = $Y_eval)
 @time pred_train_logistic = predict(model, x_train);
@@ -94,11 +94,11 @@ sqrt(mean((pred_train_logistic .- y_train) .^ 2))
 # L1
 params1 = EvoTreeRegressor(
     loss=:L1, alpha=0.5,
-    nrounds=200, nbins=64,
-    lambda=0.0, gamma=0.0, eta=0.05,
+    nrounds=500, nbins=64,
+    lambda=0.0, gamma=0.0, eta=0.1,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, metric=:mae);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:mae);
 @time pred_train_L1 = predict(model, x_train)
 @time pred_eval_L1 = predict(model, x_eval)
 sqrt(mean((pred_train_L1 .- y_train) .^ 2))
@@ -114,33 +114,33 @@ savefig("figures/regression_sinus.png")
 # Poisson
 params1 = EvoTreeCount(
     loss=:poisson,
-    nrounds=200, nbins=64,
-    lambda=0.5, gamma=0.1, eta=0.1,
+    nrounds=500, nbins=64,
+    lambda=0.1, gamma=0.1, eta=0.1,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, metric=:poisson);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:poisson);
 @time pred_train_poisson = predict(model, x_train);
 sqrt(mean((pred_train_poisson .- y_train) .^ 2))
 
 # Gamma
 params1 = EvoTreeRegressor(
     loss=:gamma,
-    nrounds=200, nbins=64,
-    lambda=0.5, gamma=0.1, eta=0.1,
+    nrounds=500, nbins=64,
+    lambda=0.1, gamma=0.1, eta=0.02,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, metric=:gamma);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:gamma);
 @time pred_train_gamma = predict(model, x_train);
 sqrt(mean((pred_train_gamma .- y_train) .^ 2))
 
 # Tweedie
 params1 = EvoTreeRegressor(
     loss=:tweedie,
-    nrounds=200, nbins=64,
+    nrounds=500, nbins=64,
     lambda=0.5, gamma=0.1, eta=0.1,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, metric=:tweedie);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:tweedie);
 @time pred_train_tweedie = predict(model, x_train);
 sqrt(mean((pred_train_tweedie .- y_train) .^ 2))
 
@@ -158,11 +158,11 @@ savefig("figures/regression_sinus2.png")
 # q50
 params1 = EvoTreeRegressor(
     loss=:quantile, alpha=0.5,
-    nrounds=200, nbins=64,
-    lambda=1.0, gamma=0.0, eta=0.05,
+    nrounds=500, nbins=64,
+    lambda=0.1, gamma=0.0, eta=0.05,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:mae);
 # 116.822 ms (74496 allocations: 36.41 MiB) for 100 iterations
 # @btime model = grow_gbtree($X_train, $Y_train, $params1, X_eval = $X_eval, Y_eval = $Y_eval)
 @time pred_train_q50 = predict(model, x_train)
@@ -171,8 +171,8 @@ sum(pred_train_q50 .< y_train) / length(y_train)
 # q20
 params1 = EvoTreeRegressor(
     loss=:quantile, alpha=0.2,
-    nrounds=200, nbins=64,
-    lambda=1.0, gamma=0.0, eta=0.05,
+    nrounds=300, nbins=64,
+    lambda=0.1, gamma=0.0, eta=0.05,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
 @time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25);
@@ -182,8 +182,8 @@ sum(pred_train_q20 .< y_train) / length(y_train)
 # q80
 params1 = EvoTreeRegressor(
     loss=:quantile, alpha=0.8,
-    nrounds=200, nbins=64,
-    lambda=1.0, gamma=0.0, eta=0.05,
+    nrounds=300, nbins=64,
+    lambda=0.1, gamma=0.0, eta=0.05,
     max_depth=6, min_weight=1.0,
     rowsample=0.5, colsample=1.0)
 @time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25)
@@ -203,12 +203,12 @@ savefig("figures/quantiles_sinus.png")
 ###############################
 params1 = EvoTreeMLE(
     loss=:gaussian,
-    nrounds=200, nbins=64,
+    nrounds=500, nbins=64,
     lambda=0.1, gamma=0.1, eta=0.05,
     max_depth=6, min_weight=1.0,
     rowsample=1.0, colsample=1.0, rng=123)
 
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=10, metric=:gaussian);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:gaussian);
 # @time model = fit_evotree(params1, X_train, Y_train, print_every_n = 10);
 @time pred_train = EvoTrees.predict(model, x_train);
 # @btime pred_train = EvoTrees.predict(model, X_train);
@@ -234,12 +234,12 @@ savefig("figures/gaussian-sinus.png")
 ###############################
 params1 = EvoTrees.EvoTreeMLE(
     loss = :logistic,
-    nrounds=200, nbins=64,
-    lambda=1.0, gamma=0.1, eta=0.05,
+    nrounds=500, nbins=64,
+    lambda=1.0, gamma=0.1, eta=0.03,
     max_depth=6, min_weight=1.0,
     rowsample=1.0, colsample=1.0, rng=123)
 
-@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=10, metric=:logistic);
+@time model = fit_evotree(params1; x_train, y_train, x_eval, y_eval, print_every_n=25, early_stopping_rounds = 50, metric=:logistic_mle);
 # @time model = fit_evotree(params1, X_train, Y_train, print_every_n = 10);
 @time pred_train = EvoTrees.predict(model, x_train);
 # @btime pred_train = EvoTrees.predict(model, X_train);