refactor: reduce build time (#4232)

* refactor: reduce build time * add back as private * update header * add missing include * add include * change format signatuer * build fix * add header * include winsock2 first * add include * include headers * fixes * las fixes * add header * Update vowpalwabbit.cpp * #define NOMINMAX * header * includes * includes * Update gen_cs_example.h
VowpalWabbit · Oct 19, 2022 · 0180673 · 0180673
1 parent 28e7a79
commit 0180673
Show file tree

Hide file tree

Showing 110 changed files with 604 additions and 407 deletions.
diff --git a/cs/cli/vowpalwabbit.cpp b/cs/cli/vowpalwabbit.cpp
@@ -7,6 +7,7 @@
 #include "vowpalwabbit.h"
 #include "vw/core/best_constant.h"
 #include "vw/core/parser.h"
+#include "vw/core/learner.h"
 #include "vw/common/hash.h"
 #include "vw_example.h"
 #include "vw_builder.h"

diff --git a/cs/cli/vw_example.cpp b/cs/cli/vw_example.cpp
@@ -2,6 +2,8 @@
 // individual contributors. All rights reserved. Released under a BSD (revised)
 // license as described in the file LICENSE.
 
+#define NOMINMAX
+
 #include "vowpalwabbit.h"
 #include "vw_example.h"
 #include "vw_prediction.h"

diff --git a/cs/cli/vw_prediction.cpp b/cs/cli/vw_prediction.cpp
@@ -6,6 +6,7 @@
 #include "vw_example.h"
 #include "vw_base.h"
 #include "vowpalwabbit.h"
+#include "vw/core/learner.h"
 
 namespace VW
 {

diff --git a/cs/cli/vw_prediction.h b/cs/cli/vw_prediction.h
@@ -6,6 +6,8 @@
 
 #include "vw_clr.h"
 
+#include "vw/core/prediction_type.h"
+
 namespace VW
 {
 ref class VowpalWabbitExample;

diff --git a/cs/vw.net.native/vw.net.workspace.cc b/cs/vw.net.native/vw.net.workspace.cc
@@ -3,6 +3,7 @@
 #include "vw/allreduce/allreduce.h"
 #include "vw/common/string_view.h"
 #include "vw/core/best_constant.h"
+#include "vw/core/learner.h"
 #include "vw/core/parse_example.h"
 #include "vw/core/shared_data.h"
 
@@ -224,7 +225,7 @@ API vw_net_native::ERROR_CODE WorkspaceRunMultiPass(
     {
       workspace->vw->do_reset_source = true;
       VW::start_parser(*workspace->vw);
-      LEARNER::generic_driver(*workspace->vw);
+      VW::LEARNER::generic_driver(*workspace->vw);
       VW::end_parser(*workspace->vw);
     }
     CATCH_RETURN_STATUS

diff --git a/cs/vw.net.native/vw.net.workspace.h b/cs/vw.net.native/vw.net.workspace.h
@@ -4,6 +4,8 @@
 #include "vw.net.stream_adapter.h"
 #include "vw/core/example.h"
 #include "vw/core/json_utils.h"
+#include "vw/core/label_type.h"
+#include "vw/core/prediction_type.h"
 #include "vw/core/vw.h"
 
 #include <string>

diff --git a/java/src/main/c++/jni_base_learner.h b/java/src/main/c++/jni_base_learner.h
@@ -2,6 +2,8 @@
 #define VW_BASE_LEARNER_H
 
 #include "util.h"
+#include "vw/core/global_data.h"
+#include "vw/core/learner.h"
 #include "vw/core/vw.h"
 
 #include <jni.h>

diff --git a/python/pylibvw.cc b/python/pylibvw.cc
@@ -10,6 +10,7 @@
 #include "vw/core/cost_sensitive.h"
 #include "vw/core/global_data.h"
 #include "vw/core/kskip_ngram_transformer.h"
+#include "vw/core/learner.h"
 #include "vw/core/merge.h"
 #include "vw/core/multiclass.h"
 #include "vw/core/multilabel.h"

diff --git a/test/tools/parser_throughput/main.cc b/test/tools/parser_throughput/main.cc
@@ -3,6 +3,7 @@
 #include "vw/config/option_group_definition.h"
 #include "vw/config/options_cli.h"
 #include "vw/core/io_buf.h"
+#include "vw/core/learner.h"
 #include "vw/core/parse_example_json.h"
 #include "vw/core/vw.h"
 #include "vw/io/io_adapter.h"

diff --git a/test/unit_test/baseline_cb_test.cc b/test/unit_test/baseline_cb_test.cc
@@ -3,6 +3,8 @@
 // license as described in the file LICENSE.
 
 #include "test_common.h"
+#include "vw/core/learner.h"
+#include "vw/core/metric_sink.h"
 #include "vw/core/rand48.h"
 #include "vw/core/vw.h"
 

diff --git a/test/unit_test/cats_tree_test.cc b/test/unit_test/cats_tree_test.cc
@@ -6,6 +6,7 @@
 
 #include "test_common.h"
 #include "vw/core/cb_label_parser.h"
+#include "vw/core/learner.h"
 #include "vw/core/simple_label.h"
 
 #include <boost/test/unit_test.hpp>

diff --git a/test/unit_test/custom_reduction_test.cc b/test/unit_test/custom_reduction_test.cc
@@ -10,6 +10,7 @@
 
 #include "vw/common/vw_exception.h"
 #include "vw/config/options.h"
+#include "vw/core/learner.h"
 #include "vw/core/reduction_stack.h"
 #include "vw/core/vw.h"
 #include "vw/core/vw_fwd.h"

diff --git a/test/unit_test/epsilon_decay_test.cc b/test/unit_test/epsilon_decay_test.cc
@@ -2,6 +2,7 @@
 
 #include "simulator.h"
 #include "test_common.h"
+#include "vw/core/learner.h"
 #include "vw/core/metric_sink.h"
 #include "vw/core/setup_base.h"
 

diff --git a/test/unit_test/feature_group_test.cc b/test/unit_test/feature_group_test.cc
@@ -5,6 +5,7 @@
 #include "vw/core/feature_group.h"
 
 #include "test_common.h"
+#include "vw/core/scope_exit.h"
 #include "vw/core/unique_sort.h"
 
 #include <boost/test/test_tools.hpp>

diff --git a/test/unit_test/interactions_test.cc b/test/unit_test/interactions_test.cc
@@ -10,6 +10,7 @@
 #include "vw/core/interactions_predict.h"
 #include "vw/core/parse_args.h"
 #include "vw/core/reductions/gd.h"
+#include "vw/core/scope_exit.h"
 #include "vw/core/vw.h"
 
 #include <array>
@@ -420,14 +421,14 @@ BOOST_AUTO_TEST_CASE(extent_interaction_expansion_test)
     VW::finish(*vw);
   });
 
-  INTERACTIONS::generate_interactions_object_cache cache;
+  VW::details::generate_interactions_object_cache cache;
 
   {
     const auto extent_terms = parse_full_name_interactions(*vw, "user_info|user_info");
     size_t counter = 0;
     INTERACTIONS::generate_generic_extent_combination_iterative(
         ex->feature_space, extent_terms,
-        [&](const std::vector<INTERACTIONS::features_range_t>& combination) {
+        [&](const std::vector<VW::details::features_range_t>& combination) {
           counter++;
           BOOST_REQUIRE_EQUAL(combination.size(), 2);
         },
@@ -440,7 +441,7 @@ BOOST_AUTO_TEST_CASE(extent_interaction_expansion_test)
     size_t counter = 0;
     INTERACTIONS::generate_generic_extent_combination_iterative(
         ex->feature_space, extent_terms,
-        [&](const std::vector<INTERACTIONS::features_range_t>& combination) {
+        [&](const std::vector<VW::details::features_range_t>& combination) {
           counter++;
           BOOST_REQUIRE_EQUAL(combination.size(), 3);
         },
@@ -453,7 +454,7 @@ BOOST_AUTO_TEST_CASE(extent_interaction_expansion_test)
     size_t counter = 0;
     INTERACTIONS::generate_generic_extent_combination_iterative(
         ex->feature_space, extent_terms,
-        [&](const std::vector<INTERACTIONS::features_range_t>& combination) {
+        [&](const std::vector<VW::details::features_range_t>& combination) {
           counter++;
           BOOST_REQUIRE_EQUAL(combination.size(), 2);
         },

diff --git a/test/unit_test/minimal_custom_reduction.cc b/test/unit_test/minimal_custom_reduction.cc
@@ -8,6 +8,7 @@
 // this test is a copy from unit_test/prediction_test.cc
 // it adds a noop reduction on top
 
+#include "vw/core/learner.h"
 #include "vw/core/reduction_stack.h"
 #include "vw/core/vw.h"
 #include "vw/core/vw_fwd.h"

diff --git a/test/unit_test/offset_tree_test.cc b/test/unit_test/offset_tree_test.cc
@@ -6,6 +6,7 @@
 
 #include "test_common.h"
 #include "vw/core/cb_label_parser.h"
+#include "vw/core/learner.h"
 
 #include <boost/test/unit_test.hpp>
 #include <utility>

diff --git a/vowpalwabbit/allreduce/include/vw/allreduce/allreduce.h b/vowpalwabbit/allreduce/include/vw/allreduce/allreduce.h
@@ -45,6 +45,7 @@ using socket_t = int;
 #  include <future>
 #endif
 
+#include "vw/allreduce/allreduce_type.h"
 #include "vw/common/future_compat.h"
 #include "vw/common/vw_exception.h"
 #include "vw/io/logger.h"
@@ -94,12 +95,6 @@ void addbufs(T* buf1, const T* buf2, const size_t n)
 
 }  // namespace details
 
-enum class all_reduce_type
-{
-  SOCKET,
-  THREAD
-};
-
 class all_reduce_base
 {
 public:

diff --git a/vowpalwabbit/allreduce/include/vw/allreduce/allreduce_type.h b/vowpalwabbit/allreduce/include/vw/allreduce/allreduce_type.h
@@ -0,0 +1,14 @@
+// Copyright (c) by respective owners including Yahoo!, Microsoft, and
+// individual contributors. All rights reserved. Released under a BSD (revised)
+// license as described in the file LICENSE.
+
+#pragma once
+
+namespace VW
+{
+enum class all_reduce_type
+{
+  SOCKET,
+  THREAD
+};
+}  // namespace VW
diff --git a/vowpalwabbit/c_wrapper/src/vwdll.cc b/vowpalwabbit/c_wrapper/src/vwdll.cc
@@ -4,6 +4,7 @@
 
 #include "vw/c_wrapper/vwdll.h"
 
+#include "vw/core/learner.h"
 #include "vw/core/memory.h"
 #include "vw/core/parse_args.h"
 #include "vw/core/parser.h"

diff --git a/vowpalwabbit/cli/src/main.cc b/vowpalwabbit/cli/src/main.cc
@@ -6,6 +6,7 @@
 #include "vw/config/options.h"
 #include "vw/config/options_cli.h"
 #include "vw/core/global_data.h"
+#include "vw/core/learner.h"
 #include "vw/core/memory.h"
 #include "vw/core/vw.h"
 #include "vw/io/logger.h"

diff --git a/vowpalwabbit/core/CMakeLists.txt b/vowpalwabbit/core/CMakeLists.txt
@@ -12,11 +12,12 @@ set(vw_core_headers
   include/vw/core/cached_learner.h
   include/vw/core/cb_continuous_label.h
   include/vw/core/cb_label_parser.h
+  include/vw/core/multi_ex.h
+  include/vw/core/learner_fwd.h
   include/vw/core/cb_type.h
   include/vw/core/cb.h
   include/vw/core/ccb_label.h
   include/vw/core/ccb_reduction_features.h
-  include/vw/core/compat.h
   include/vw/core/confidence_sequence.h
   include/vw/core/constant.h
   include/vw/core/continuous_actions_reduction_features.h
@@ -366,7 +367,7 @@ vw_add_library(
     NAME "core"
     TYPE "STATIC_ONLY"
     SOURCES ${vw_core_headers} ${vw_core_sources}
-    PUBLIC_DEPS vw_common vw_explore vw_allreduce vw_config ${spdlog_target} fmt::fmt
+    PUBLIC_DEPS vw_common vw_explore vw_allreduce vw_config fmt::fmt
     PRIVATE_DEPS
         # Special case for dealing with header-only INTERFACE dependencies of a STATIC library
         # Use BUILD_INTERFACE to prevent them from being exported, i.e. treat them as PRIVATE

diff --git a/vowpalwabbit/core/include/vw/core/accumulate.h b/vowpalwabbit/core/include/vw/core/accumulate.h
@@ -8,6 +8,7 @@
 #include "vw/core/vw_fwd.h"
 
 #include <cstddef>
+#include <cstdint>
 
 void accumulate(VW::workspace& all, parameters& weights, size_t o);
 float accumulate_scalar(VW::workspace& all, float local_sum);

diff --git a/vowpalwabbit/core/include/vw/core/cache.h b/vowpalwabbit/core/include/vw/core/cache.h
@@ -4,6 +4,8 @@
 
 #pragma once
 
+#include "vw/core/multi_ex.h"
+#include "vw/core/v_array.h"
 #include "vw/core/vw_fwd.h"
 
 namespace VW

diff --git a/vowpalwabbit/core/include/vw/core/ccb_label.h b/vowpalwabbit/core/include/vw/core/ccb_label.h
@@ -11,7 +11,6 @@
 #include "vw/core/label_parser.h"
 #include "vw/core/v_array.h"
 
-#include <fmt/core.h>
 #include <fmt/format.h>
 
 #include <cstdint>

diff --git a/vowpalwabbit/core/include/vw/core/compat.h b/vowpalwabbit/core/include/vw/core/compat.h
diff --git a/vowpalwabbit/core/include/vw/core/cost_sensitive.h b/vowpalwabbit/core/include/vw/core/cost_sensitive.h
@@ -4,6 +4,7 @@
 #pragma once
 
 #include "vw/core/label_parser.h"
+#include "vw/core/multi_ex.h"
 #include "vw/core/vw_fwd.h"
 
 #include <cstdint>

diff --git a/vowpalwabbit/core/include/vw/core/decision_scores.h b/vowpalwabbit/core/include/vw/core/decision_scores.h
@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include "vw/core/action_score.h"
 #include "vw/core/constant.h"
 #include "vw/core/v_array.h"
 

diff --git a/vowpalwabbit/core/include/vw/core/example.h b/vowpalwabbit/core/include/vw/core/example.h
@@ -173,8 +173,6 @@ inline void add_passthrough_feature_magic(example& ec, uint64_t magic, uint64_t
 #define add_passthrough_feature(ec, i, x) \
   VW::add_passthrough_feature_magic(ec, __FILE__[0] * 483901 + __FILE__[1] * 3417 + __FILE__[2] * 8490177, i, x);
 
-using multi_ex = std::vector<example*>;
-
 void return_multiple_example(VW::workspace& all, VW::multi_ex& examples);
 
 using example_factory_t = example& (*)(void*);

diff --git a/vowpalwabbit/core/include/vw/core/gd_predict.h b/vowpalwabbit/core/include/vw/core/gd_predict.h
@@ -51,8 +51,8 @@ template <class DataT, class WeightOrIndexT, void (*FuncT)(DataT&, float, Weight
 inline void generate_interactions(const std::vector<std::vector<VW::namespace_index>>& interactions,
     const std::vector<std::vector<extent_term>>& extent_interactions, bool permutations, VW::example_predict& ec,
     DataT& dat, WeightsT& weights, size_t& num_interacted_features,
-    INTERACTIONS::generate_interactions_object_cache& cache)  // default value removed to eliminate
-                                                              // ambiguity in old complers
+    VW::details::generate_interactions_object_cache& cache)  // default value removed to eliminate
+                                                             // ambiguity in old complers
 {
   INTERACTIONS::generate_interactions<DataT, WeightOrIndexT, FuncT, false, dummy_func<DataT>, WeightsT>(
       interactions, extent_interactions, permutations, ec, dat, weights, num_interacted_features, cache);
@@ -65,7 +65,7 @@ inline void foreach_feature(WeightsT& weights, bool ignore_some_linear,
     std::array<bool, VW::NUM_NAMESPACES>& ignore_linear,
     const std::vector<std::vector<VW::namespace_index>>& interactions,
     const std::vector<std::vector<extent_term>>& extent_interactions, bool permutations, VW::example_predict& ec,
-    DataT& dat, size_t& num_interacted_features, INTERACTIONS::generate_interactions_object_cache& cache)
+    DataT& dat, size_t& num_interacted_features, VW::details::generate_interactions_object_cache& cache)
 {
   uint64_t offset = ec.ft_offset;
   if (ignore_some_linear)
@@ -93,7 +93,7 @@ inline void foreach_feature(WeightsT& weights, bool ignore_some_linear,
     std::array<bool, VW::NUM_NAMESPACES>& ignore_linear,
     const std::vector<std::vector<VW::namespace_index>>& interactions,
     const std::vector<std::vector<extent_term>>& extent_interactions, bool permutations, VW::example_predict& ec,
-    DataT& dat, INTERACTIONS::generate_interactions_object_cache& cache)
+    DataT& dat, VW::details::generate_interactions_object_cache& cache)
 {
   size_t num_interacted_features_ignored = 0;
   foreach_feature<DataT, WeightOrIndexT, FuncT, WeightsT>(weights, ignore_some_linear, ignore_linear, interactions,
@@ -107,7 +107,7 @@ inline float inline_predict(WeightsT& weights, bool ignore_some_linear,
     std::array<bool, VW::NUM_NAMESPACES>& ignore_linear,
     const std::vector<std::vector<VW::namespace_index>>& interactions,
     const std::vector<std::vector<extent_term>>& extent_interactions, bool permutations, VW::example_predict& ec,
-    INTERACTIONS::generate_interactions_object_cache& cache, float initial = 0.f)
+    VW::details::generate_interactions_object_cache& cache, float initial = 0.f)
 {
   foreach_feature<float, float, vec_add, WeightsT>(
       weights, ignore_some_linear, ignore_linear, interactions, extent_interactions, permutations, ec, initial, cache);
@@ -119,7 +119,7 @@ inline float inline_predict(WeightsT& weights, bool ignore_some_linear,
     std::array<bool, VW::NUM_NAMESPACES>& ignore_linear,
     const std::vector<std::vector<VW::namespace_index>>& interactions,
     const std::vector<std::vector<extent_term>>& extent_interactions, bool permutations, VW::example_predict& ec,
-    size_t& num_interacted_features, INTERACTIONS::generate_interactions_object_cache& cache, float initial = 0.f)
+    size_t& num_interacted_features, VW::details::generate_interactions_object_cache& cache, float initial = 0.f)
 {
   foreach_feature<float, float, vec_add, WeightsT>(weights, ignore_some_linear, ignore_linear, interactions,
       extent_interactions, permutations, ec, initial, num_interacted_features, cache);

diff --git a/vowpalwabbit/core/include/vw/core/gen_cs_example.h b/vowpalwabbit/core/include/vw/core/gen_cs_example.h
@@ -5,7 +5,6 @@
 #include "vw/common/vw_exception.h"
 #include "vw/core/reductions/cb/cb_algs.h"
 #include "vw/core/scope_exit.h"
-#include "vw/core/vw.h"
 
 #include <cfloat>