Improve build time

- Fix compilation errors with clang-18 due to Werror - Run include-what-you-use to reduce includes and add forward declarations
man-group · Apr 25, 2024 · f3cde62 · f3cde62
1 parent a5c1741
commit f3cde62
Show file tree

Hide file tree

Showing 203 changed files with 478 additions and 862 deletions.
diff --git a/cpp/arcticdb/CMakeLists.txt b/cpp/arcticdb/CMakeLists.txt
@@ -433,10 +433,6 @@ set(arcticdb_srcs
         processing/operation_dispatch.cpp
         processing/operation_dispatch_unary.cpp
         processing/operation_dispatch_binary.cpp
-        processing/operation_dispatch_binary_eq.cpp
-        processing/operation_dispatch_binary_gt.cpp
-        processing/operation_dispatch_binary_lt.cpp
-        processing/operation_dispatch_binary_operator.cpp
         python/python_to_tensor_frame.cpp
         storage/config_resolvers.cpp
         storage/failure_simulation.cpp
@@ -797,7 +793,9 @@ if(${TEST})
             storage/test/test_storage_factory.cpp
             storage/test/test_storage_exceptions.cpp
             storage/test/test_azure_storage.cpp
+            stream/test/stream_test_common.hpp
             stream/test/stream_test_common.cpp
+            stream/test/test_store_common.hpp
             stream/test/test_aggregator.cpp
             stream/test/test_append_map.cpp
             stream/test/test_row_builder.cpp
@@ -917,13 +915,13 @@ if(${TEST})
 
     target_link_libraries(benchmarks
             PUBLIC
-            benchmark::benchmark
-            benchmark::benchmark_main
-            ${COMMON_PUBLIC_TEST_LIBRARIES}
+                benchmark::benchmark
+                benchmark::benchmark_main
             PRIVATE
-            ${AWSSDK_LINK_LIBRARIES}
-            arcticdb_core_static
-            )
+                arcticdb_core_static
+                Python::Python
+                ${AWSSDK_LINK_LIBRARIES}
+    )
 
     set(rapidcheck_srcs
             column_store/test/rapidcheck_column_store.cpp

diff --git a/cpp/arcticdb/async/async_store.hpp b/cpp/arcticdb/async/async_store.hpp
@@ -10,11 +10,9 @@
 #include <arcticdb/async/task_scheduler.hpp>
 #include <arcticdb/util/clock.hpp>
 #include <arcticdb/storage/store.hpp>
-#include <arcticdb/entity/performance_tracing.hpp>
 #include <arcticdb/storage/library.hpp>
 #include <folly/futures/Future.h>
 #include <arcticdb/async/tasks.hpp>
-#include <arcticdb/stream/stream_utils.hpp>
 #include <arcticdb/processing/clause.hpp>
 #include <arcticdb/storage/key_segment_pair.hpp>
 
@@ -350,7 +348,7 @@ std::vector<folly::Future<bool>> batch_key_exists(
                 std::move(seg),
                 codec_,
                 encoding_version_}();
-            return std::pair<storage::KeySegmentPair, FrameSlice>(std::move(key_seg), std::move(slice));
+            return std::pair<storage::KeySegmentPair, pipelines::FrameSlice>(std::move(key_seg), std::move(slice));
         })
         .thenValue([de_dup_map](auto &&ks) -> std::pair<KeyOptSegment, pipelines::FrameSlice> {
             auto [key_seg, slice] = std::forward<decltype(ks)>(ks);

diff --git a/cpp/arcticdb/async/python_bindings.cpp b/cpp/arcticdb/async/python_bindings.cpp
@@ -5,7 +5,6 @@
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
 
-#include <arcticdb/python/python_utils.hpp>
 #include <arcticdb/async/python_bindings.hpp>
 #include <arcticdb/async/task_scheduler.hpp>
 

diff --git a/cpp/arcticdb/async/task_scheduler.hpp b/cpp/arcticdb/async/task_scheduler.hpp
@@ -7,12 +7,7 @@
 
 #pragma once
 
-#include <arcticdb/entity/atom_key.hpp>
-#include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/hash.hpp>
-#include <arcticdb/util/exponential_backoff.hpp>
 #include <arcticdb/util/configs_map.hpp>
-#include <arcticdb/util/home_directory.hpp>
 #include <arcticdb/async/base_task.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
 
@@ -25,7 +20,6 @@
 #include <filesystem>
 #include <string>
 #include <fstream>
-#include <fmt/format.h>
 #include <type_traits>
 
 namespace arcticdb::async {

diff --git a/cpp/arcticdb/async/tasks.cpp b/cpp/arcticdb/async/tasks.cpp
@@ -6,7 +6,6 @@
  */
 
 #include <arcticdb/async/tasks.hpp>
-#include <arcticdb/pipeline/read_frame.hpp>
 
 namespace arcticdb::async {
 

diff --git a/cpp/arcticdb/async/tasks.hpp b/cpp/arcticdb/async/tasks.hpp
@@ -11,16 +11,13 @@
 #include <arcticdb/storage/library.hpp>
 #include <arcticdb/storage/storage_options.hpp>
 #include <arcticdb/entity/types.hpp>
-#include <arcticdb/util/hash.hpp>
 #include <arcticdb/stream/stream_utils.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
 #include <arcticdb/stream/stream_source.hpp>
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/entity/variant_key.hpp>
 #include <arcticdb/stream/stream_sink.hpp>
 #include <arcticdb/async/base_task.hpp>
 #include <arcticdb/pipeline/frame_slice.hpp>
-#include <arcticdb/processing/processing_unit.hpp>
 #include <arcticdb/util/constructors.hpp>
 #include <arcticdb/codec/codec.hpp>
 

diff --git a/cpp/arcticdb/async/test/test_async.cpp b/cpp/arcticdb/async/test/test_async.cpp
@@ -9,13 +9,10 @@
 #include <arcticdb/storage/common.hpp>
 #include <arcticdb/storage/config_resolvers.hpp>
 #include <arcticdb/storage/library_index.hpp>
-#include <arcticdb/storage/storage_factory.hpp>
 #include <arcticdb/async/async_store.hpp>
 #include <arcticdb/util/test/config_common.hpp>
 #include <arcticdb/pipeline/frame_slice.hpp>
-#include <arcticdb/util/random.h>
 
-#include <fmt/format.h>
 
 #include <string>
 #include <vector>
@@ -43,7 +40,7 @@ TEST(Async, SinkBasic) {
 
     auto seg = ac::SegmentInMemory();
     aa::EncodeAtomTask enc{
-        ac::entity::KeyType::GENERATION, ac::entity::VersionId{6}, ac::entity::NumericId{123}, ac::entity::NumericId{456}, ac::timestamp{457}, {ac::entity::NumericIndex{999}}, std::move(seg), codec_opt, ac::EncodingVersion::V2
+        ac::entity::KeyType::GENERATION, ac::entity::VersionId{6}, ac::entity::NumericId{123}, ac::entity::NumericId{456}, ac::timestamp{457}, ac::entity::NumericIndex{999}, std::move(seg), codec_opt, ac::EncodingVersion::V2
     };
 
     auto v = sched.submit_cpu_task(std::move(enc)).via(&aa::io_executor()).thenValue(aa::WriteSegmentTask{lib}).get();

diff --git a/cpp/arcticdb/codec/codec-inl.hpp b/cpp/arcticdb/codec/codec-inl.hpp
@@ -13,7 +13,6 @@
 #include <arcticdb/codec/passthrough.hpp>
 #include <arcticdb/codec/zstd.hpp>
 #include <arcticdb/codec/lz4.hpp>
-#include <arcticdb/codec/encoded_field.hpp>
 #include <arcticdb/codec/magic_words.hpp>
 
 #include <arcticdb/util/bitset.hpp>

diff --git a/cpp/arcticdb/codec/codec.cpp b/cpp/arcticdb/codec/codec.cpp
@@ -6,16 +6,13 @@
  */
 #include <arcticdb/codec/codec.hpp>
 #include <arcticdb/column_store/string_pool.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
-#include <arcticdb/codec/default_codecs.hpp>
 #include <arcticdb/codec/encoded_field.hpp>
 #include <arcticdb/codec/encoded_field_collection.hpp>
-
+#include <arcticdb/column_store/memory_segment.hpp>
+#include <arcticdb/codec/segment.hpp>
 
 #include <string>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
-
 
 #include <arcticdb/codec/encode_common.hpp>
 

diff --git a/cpp/arcticdb/codec/codec.hpp b/cpp/arcticdb/codec/codec.hpp
@@ -7,16 +7,16 @@
 
 #pragma once
 
-#include <arcticdb/codec/core.hpp>
-#include <arcticdb/codec/segment.hpp>
-#include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/entity/types.hpp>
 #include <arcticdb/codec/encode_common.hpp>
 
 namespace arcticdb {
 
 using ShapesBlockTDT = TypeDescriptorTag<DataTypeTag<DataType::INT64>, DimensionTag<Dimension::Dim0>>;
 
+class SegmentInMemory;
+class Segment;
+
 Segment encode_dispatch(
     SegmentInMemory&& in_mem_seg,
     const arcticdb::proto::encoding::VariantCodec &codec_opts,

diff --git a/cpp/arcticdb/codec/encoding_sizes.cpp b/cpp/arcticdb/codec/encoding_sizes.cpp
@@ -5,9 +5,8 @@
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
 
-#include <arcticdb/codec/encoding_sizes.hpp>
-#include <arcticdb/entity/types.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
+#include <arcticdb/entity/protobufs.hpp>
+#include <arcticdb/entity/types_proto.hpp>
 
 namespace arcticdb::encoding_sizes {
 

diff --git a/cpp/arcticdb/codec/encoding_sizes.hpp b/cpp/arcticdb/codec/encoding_sizes.hpp
@@ -7,11 +7,11 @@
 
 #pragma once
 
-#include <arcticdb/entity/protobufs.hpp>
 #include <arcticdb/util/pb_util.hpp>
 #include <arcticdb/log/log.hpp>
 #include <arcticdb/util/preconditions.hpp>
 #include <numeric>
+#include <arcticdb/entity/protobufs.hpp>
 
 namespace arcticdb::encoding_sizes {
 

diff --git a/cpp/arcticdb/codec/python_bindings.cpp b/cpp/arcticdb/codec/python_bindings.cpp
@@ -13,16 +13,12 @@
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/entity/protobufs.hpp>
 #include <arcticdb/python/python_utils.hpp>
-#include <arcticdb/util/flatten_utils.hpp>
 #include <arcticdb/util/cursored_buffer.hpp>
 #include <arcticdb/util/preconditions.hpp>
 #include <arcticdb/util/pybind_mutex.hpp>
 
-#include <fmt/format.h>
-
 #include <memory>
 #include <algorithm>
-#include <string>
 
 namespace py = pybind11;
 using namespace arcticdb::python_util;

diff --git a/cpp/arcticdb/codec/segment.cpp b/cpp/arcticdb/codec/segment.cpp
@@ -7,12 +7,9 @@
 
 #include <arcticdb/codec/segment.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
-#include <arcticdb/stream/protobuf_mappings.hpp>
 
-#include <arcticdb/util/pb_util.hpp>
 #include <arcticdb/util/dump_bytes.hpp>
 #include <arcticdb/codec/codec.hpp>
-#include <arcticdb/codec/magic_words.hpp>
 
 namespace arcticdb {
 namespace segment_size {

diff --git a/cpp/arcticdb/codec/segment.hpp b/cpp/arcticdb/codec/segment.hpp
@@ -7,12 +7,9 @@
 
 #pragma once
 
-#include "util/buffer.hpp"
+#include <arcticdb/util/buffer.hpp>
 #include <arcticdb/storage/common.hpp>
-#include <codec/encoding_sizes.hpp>
-#include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/arena.h>
-#include <util/buffer_pool.hpp>
 #include <arcticdb/entity/field_collection.hpp>
 
 #include <iostream>

diff --git a/cpp/arcticdb/codec/slice_data_sink.hpp b/cpp/arcticdb/codec/slice_data_sink.hpp
@@ -16,9 +16,9 @@ struct SliceDataSink {
     SliceDataSink(uint8_t *data, std::size_t size) : data_(data), shape_(0), current_size_(0), total_size_(size) {
     }
 
-    shape_t *allocate_shapes(std::size_t s) ARCTICDB_UNUSED {
+    entity::shape_t *allocate_shapes(std::size_t s) ARCTICDB_UNUSED {
         if (s == 0) return nullptr;
-        util::check_arg(s == 8, "expected exactly one shape, actual {}", s / sizeof(shape_t));
+        util::check_arg(s == 8, "expected exactly one shape, actual {}", s / sizeof(entity::shape_t));
         return &shape_;
     }
 
@@ -37,7 +37,7 @@ struct SliceDataSink {
 
 private:
     uint8_t *data_;
-    shape_t shape_;
+    entity::shape_t shape_;
     std::size_t current_size_;
     std::size_t total_size_;
 };

diff --git a/cpp/arcticdb/codec/test/test_codec.cpp b/cpp/arcticdb/codec/test/test_codec.cpp
@@ -11,7 +11,6 @@
 #include <arcticdb/util/test/test_utils.hpp>
 #include <arcticdb/util/test/generators.hpp>
 #include <arcticdb/util/random.h>
-#include <arcticdb/stream/row_builder.hpp>
 #include <arcticdb/stream/aggregator.hpp>
 #include <arcticdb/codec/typed_block_encoder_impl.hpp>
 

diff --git a/cpp/arcticdb/column_store/chunked_buffer.hpp b/cpp/arcticdb/column_store/chunked_buffer.hpp
@@ -9,15 +9,16 @@
 #pragma once
 
 #include <arcticdb/util/preconditions.hpp>
-#include <arcticdb/util/magic_num.hpp>
 #include <arcticdb/util/allocator.hpp>
 #include <arcticdb/util/constructors.hpp>
 #include <arcticdb/column_store/block.hpp>
 
-#include <boost/container/small_vector.hpp>
-
 #include <cstdint>
 
+#ifndef DEBUG_BUILD
+#include <boost/container/small_vector.hpp>
+#endif
+
 namespace arcticdb {
 
 /*

diff --git a/cpp/arcticdb/column_store/column.hpp b/cpp/arcticdb/column_store/column.hpp
@@ -19,13 +19,10 @@
 #include <arcticdb/util/preconditions.hpp>
 #include <arcticdb/util/sparse_utils.hpp>
 
-#include <folly/container/Enumerate.h>
 // Compilation fails on Mac if cstdio is not included prior to folly/Function.h due to a missing definition of memalign in folly/Memory.h
 #ifdef __APPLE__
 #include <cstdio>
 #endif
-#include <folly/Function.h>
-#include <pybind11/pybind11.h>
 #include <pybind11/numpy.h>
 
 #include <concepts>

diff --git a/cpp/arcticdb/column_store/column_utils.hpp b/cpp/arcticdb/column_store/column_utils.hpp
@@ -9,7 +9,7 @@
 
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/pipeline/frame_data_wrapper.hpp>
-
+#include <arcticdb/util/memory_tracing.hpp>
 #include <pybind11/pybind11.h>
 
 namespace arcticdb::detail {

diff --git a/cpp/arcticdb/column_store/memory_segment_impl.cpp b/cpp/arcticdb/column_store/memory_segment_impl.cpp
@@ -5,12 +5,12 @@
  * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
  */
 
+#include <arcticdb/column_store/column_map.hpp>
 #include <arcticdb/column_store/memory_segment_impl.hpp>
 #include <arcticdb/column_store/string_pool.hpp>
 #include <arcticdb/entity/type_utils.hpp>
 #include <arcticdb/pipeline/string_pool_utils.hpp>
 
-#include <google/protobuf/any.h>
 #include <google/protobuf/any.pb.h>
 
 

diff --git a/cpp/arcticdb/column_store/memory_segment_impl.hpp b/cpp/arcticdb/column_store/memory_segment_impl.hpp
@@ -11,12 +11,12 @@
 #include <arcticdb/column_store/column.hpp>
 #include <arcticdb/util/offset_string.hpp>
 #include <arcticdb/util/preconditions.hpp>
+#include <arcticdb/column_store/string_pool.hpp>
 
 #include <arcticdb/entity/timeseries_descriptor.hpp>
 #include <arcticdb/entity/performance_tracing.hpp>
 #include <arcticdb/util/magic_num.hpp>
 #include <arcticdb/util/constructors.hpp>
-#include <arcticdb/column_store/column_map.hpp>
 #include <arcticdb/entity/stream_descriptor.hpp>
 
 #include <boost/iterator/iterator_facade.hpp>
@@ -28,7 +28,7 @@ namespace google::protobuf
 }
 
 namespace arcticdb {
-
+class ColumnMap;
 class SegmentInMemoryImpl;
 
 namespace {

diff --git a/cpp/arcticdb/column_store/python_bindings.cpp b/cpp/arcticdb/column_store/python_bindings.cpp
@@ -10,7 +10,6 @@
 #include <arcticdb/column_store/column.hpp>
 #include <arcticdb/column_store/column_data.hpp>
 #include <arcticdb/column_store/string_pool.hpp>
-#include <arcticdb/python/python_utils.hpp>
 
 namespace py = pybind11;
 

diff --git a/cpp/arcticdb/column_store/string_pool.cpp b/cpp/arcticdb/column_store/string_pool.cpp
@@ -10,8 +10,6 @@
 #include <arcticdb/column_store/segment_utils.hpp>
 #include <ankerl/unordered_dense.h>
 
-#include <pybind11/pybind11.h>
-
 namespace arcticdb {
 
 /*****************

diff --git a/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp b/cpp/arcticdb/column_store/test/benchmark_memory_segment.cpp
@@ -9,7 +9,6 @@
 
 #include <arcticdb/column_store/memory_segment.hpp>
 #include <arcticdb/stream/test/stream_test_common.hpp>
-#include <folly/container/Enumerate.h>
 
 #include <algorithm>
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,7 +6,6 @@ @@
      */
     #include <arcticdb/async/tasks.hpp>
-    #include <arcticdb/pipeline/read_frame.hpp>
     namespace arcticdb::async {
@@ Expand Down @@