cleanup high depth perf improvements

Project.toml

@@ -1,7 +1,7 @@
 name = "EvoTrees"
 uuid = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5"
 authors = ["jeremiedb <[email protected]>"]
-version = "0.15.1"
+version = "0.15.2"
 
 [deps]
 BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"

README.md

@@ -46,27 +46,27 @@ Code to reproduce is availabe in [`benchmarks/regressor.jl`](https://github.com/
     - Julia: v1.9.1.
 - Algorithms
     - XGBoost: v2.3.0 (Using the `hist` algorithm).
-    - EvoTrees: v0.15.0.
+    - EvoTrees: v0.15.2.
 
 ### Training: 
 
 | Dimensions   / Algo | XGBoost CPU | EvoTrees CPU | XGBoost GPU | EvoTrees GPU |
 |---------------------|:-----------:|:------------:|:-----------:|:------------:|
-| 100K x 100          |    2.33s    |     1.09s    |    0.90s    |     2.72s    |
-| 500K x 100          |    10.7s    |     2.96s    |    1.84s    |     3.65s    |
-| 1M x 100            |    20.9s    |     6.48s    |    3.10s    |     4.45s    |
-| 5M x 100            |    108s     |     35.8s    |    12.9s    |     12.7s    |
-| 10M x 100           |    216s     |     71.6s    |    25.5s    |     23.0s    |
+| 100K x 100          |    2.34s    |     1.01s    |    0.90s    |     2.61s    |
+| 500K x 100          |    10.7s    |     3.95s    |    1.84s    |     3.41s    |
+| 1M x 100            |    21.1s    |     6.57s    |    3.10s    |     4.47s    |
+| 5M x 100            |    108s     |     36.1s    |    12.9s    |     12.5s    |
+| 10M x 100           |    218s     |     72.6s    |    25.5s    |     23.0s    |
 
 ### Inference:
 
 | Dimensions   / Algo | XGBoost CPU  | EvoTrees CPU | XGBoost GPU | EvoTrees GPU |
 |---------------------|:------------:|:------------:|:-----------:|:------------:|
-| 100K x 100          |    0.151s    |    0.053s    |     NA      |    0.036s    |
-| 500K x 100          |    0.628s    |    0.276s    |     NA      |    0.169s    |
-| 1M x 100            |    1.26s     |    0.558s    |     NA      |    0.334s    |
+| 100K x 100          |    0.151s    |    0.058s    |     NA      |    0.045s    |
+| 500K x 100          |    0.647s    |    0.248s    |     NA      |    0.172s    |
+| 1M x 100            |    1.26s     |    0.573s    |     NA      |    0.327s    |
 | 5M x 100            |    6.04s     |    2.87s     |     NA      |    1.66s     |
-| 10M x 100           |    12.4s     |    5.71s     |     NA      |    3.31s     |
+| 10M x 100           |    12.4s     |    5.71s     |     NA      |    3.40s     |
 
 ## MLJ Integration
 

benchmarks/regressor.jl

@@ -8,12 +8,17 @@ using BenchmarkTools
 using Random: seed!
 import CUDA
 
+### v.0.15.1
+# desktop | 1e6 | depth 11 | cpu: 37.2s
+# desktop | 10e6 | depth 11 | cpu
+
+### perf depth
 # desktop | 1e6 | depth 11 | cpu: 28s gpu: 73 sec  | xgboost: 26s
 # desktop | 10e6 | depth 11 | cpu 205s gpu: 109 sec | xgboost 260s
 nobs = Int(1e6)
 num_feat = Int(100)
 nrounds = 200
-max_depth = 11
+max_depth = 6
 tree_type = "binary"
 T = Float64
 nthread = Base.Threads.nthreads()
@@ -120,14 +125,11 @@ device = "cpu"
 # @time m_evo = fit_evotree(params_evo; x_train, y_train, device, verbosity, print_every_n=100);
 @info "train - eval"
 @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100);
-# using Plots
-# plot(m_evo, 2)
-
 @time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_eval=y_train, metric=metric_evo, device, verbosity, print_every_n=100);
 @info "predict"
 @time pred_evo = m_evo(x_train);
 @time pred_evo = m_evo(x_train);
-@btime m_evo($x_train);
+# @btime m_evo($x_train);
 
 @info "EvoTrees GPU"
 device = "gpu"
@@ -142,4 +144,4 @@ CUDA.@time m_evo = fit_evotree(params_evo; x_train, y_train, x_eval=x_train, y_e
 @info "predict"
 CUDA.@time pred_evo = m_evo(x_train; device);
 CUDA.@time pred_evo = m_evo(x_train; device);
-@btime m_evo($x_train; device);
+# @btime m_evo($x_train; device);

experiments/readme_plots_cpu.jl

@@ -90,7 +90,7 @@ params1 = EvoTreeRegressor(;
     nbins=64,
     lambda=0.1,
     gamma=0.1,
-    eta=0.05,
+    eta=0.1,
     max_depth=6,
     min_weight=1.0,
     rowsample=0.5,
@@ -132,7 +132,7 @@ params1 = EvoTreeRegressor(;
     nbins=64,
     lambda=0.1,
     gamma=0.1,
-    eta=0.05,
+    eta=0.1,
     max_depth=6,
     min_weight=1.0,
     rowsample=0.5,
@@ -288,7 +288,7 @@ params1 = EvoTreeRegressor(;
     loss=:tweedie,
     nrounds=500,
     nbins=64,
-    lambda=0.5,
+    lambda=0.1,
     gamma=0.1,
     eta=0.1,
     max_depth=6,
@@ -359,7 +359,7 @@ params1 = EvoTreeRegressor(;
     nbins=64,
     lambda=0.1,
     gamma=0.0,
-    eta=0.05,
+    eta=0.1,
     max_depth=6,
     min_weight=1.0,
     rowsample=0.5,
@@ -389,7 +389,7 @@ params1 = EvoTreeRegressor(;
     nbins=64,
     lambda=0.1,
     gamma=0.0,
-    eta=0.05,
+    eta=0.1,
     max_depth=6,
     min_weight=1.0,
     rowsample=0.5,
@@ -408,7 +408,7 @@ params1 = EvoTreeRegressor(;
     nbins=64,
     lambda=0.1,
     gamma=0.0,
-    eta=0.05,
+    eta=0.1,
     max_depth=6,
     min_weight=1.0,
     rowsample=0.5,
@@ -466,10 +466,10 @@ params1 = EvoTreeMLE(;
     nbins=64,
     lambda=0.1,
     gamma=0.1,
-    eta=0.05,
+    eta=0.1,
     max_depth=6,
-    min_weight=10.0,
-    rowsample=1.0,
+    min_weight=10,
+    rowsample=0.5,
     colsample=1.0,
     rng=123,
     tree_type,
@@ -549,12 +549,12 @@ params1 = EvoTrees.EvoTreeMLE(;
     loss=:logistic,
     nrounds=500,
     nbins=64,
-    lambda=1.0,
+    lambda=0.1,
     gamma=0.1,
-    eta=0.03,
+    eta=0.1,
     max_depth=6,
-    min_weight=1.0,
-    rowsample=1.0,
+    min_weight=10,
+    rowsample=0.5,
     colsample=1.0,
     tree_type,
     rng=123,

experiments/readme_plots_gpu.jl

@@ -249,7 +249,7 @@ params1 = EvoTreeGaussian(;
     gamma=0.1,
     eta=0.1,
     max_depth=6,
-    min_weight=20,
+    min_weight=10,
     rowsample=0.5,
     colsample=1.0,
     rng=123,

src/fit-utils.jl

@@ -172,58 +172,32 @@ function split_set_threads!(
     lefts = zeros(Int, nblocks)
     rights = zeros(Int, nblocks)
 
-    if nblocks == 1
-        lefts[1], rights[1] = split_set_chunk!(
-            left,
-            right,
-            is,
-            1,
-            nblocks,
-            x_bin,
-            feat,
-            cond_bin,
-            feattype,
-            offset,
-            chunk_size,
-        )
-    else
-        @threads :static for bid = 1:nblocks
-            lefts[bid], rights[bid] = split_set_chunk!(
-                left,
-                right,
-                is,
-                bid,
-                nblocks,
-                x_bin,
-                feat,
-                cond_bin,
-                feattype,
-                offset,
-                chunk_size,
-            )
+    @sync begin
+        for bid = 1:nblocks
+            @spawn begin
+                lefts[bid], rights[bid] = split_set_chunk!(
+                    left,
+                    right,
+                    is,
+                    bid,
+                    nblocks,
+                    x_bin,
+                    feat,
+                    cond_bin,
+                    feattype,
+                    offset,
+                    chunk_size,
+                )
+            end
         end
     end
 
     sum_lefts = sum(lefts)
     cumsum_lefts = cumsum(lefts)
     cumsum_rights = cumsum(rights)
-    if nblocks == 1
-        split_views_kernel!(
-            out,
-            left,
-            right,
-            1,
-            offset,
-            chunk_size,
-            lefts,
-            rights,
-            sum_lefts,
-            cumsum_lefts,
-            cumsum_rights,
-        )
-    else
-        @threads :static for bid = 1:nblocks
-            split_views_kernel!(
+    @sync begin
+        for bid = 1:nblocks
+            @spawn split_views_kernel!(
                 out,
                 left,
                 right,
@@ -258,16 +232,6 @@ function update_hist!(
     is::AbstractVector,
     js::AbstractVector,
 ) where {L<:GradientRegression}
-    # if length(is) < 1_000
-    #     for j in js
-    #         @inbounds @simd for i in is
-    #             bin = x_bin[i, j]
-    #             hist[j][1, bin] += ∇[1, i]
-    #             hist[j][2, bin] += ∇[2, i]
-    #             hist[j][3, bin] += ∇[3, i]
-    #         end
-    #     end
-    # else
     @threads :static for j in js
         @inbounds @simd for i in is
             bin = x_bin[i, j]
@@ -276,7 +240,6 @@ function update_hist!(
             hist[j][3, bin] += ∇[3, i]
         end
     end
-    # end
     return nothing
 end