Store the product's version number in C++ (#110)

So to embed WriterMetadata in symbol list keys for future use

.github/workflows/tag.yml

@@ -25,8 +25,7 @@ jobs:
     env:
       BASH_FUNC_tag_and_push%%: |-
         () {
-          sed -ri '/^\[metadata]/, /^\[/ s/^version[[:blank:]]*=.*/version = '$2/ setup.cfg
-          git add setup.cfg README.md
+          python3 version_io.py $2
           git status
           git diff --cached --exit-code || git commit -m "$1 v$2"
           set -x
@@ -50,6 +49,7 @@ jobs:
           git switch -C tagging ${{github.ref}}
           # Remove the build status badge from a version tag
           sed -i 's#<img src="https://github.com/man-group/ArcticDB/actions/workflows/build.yml/badge.svg"/>##' README.md || true
+          git add README.md
           tag_and_push "Tagging" ${{inputs.version}} refs/tags/v${{inputs.version}} ${{inputs.overwrite && '-f' || ''}}
 
       - name: Bump ${{github.ref_name}} to the next version

cpp/arcticdb/CMakeLists.txt

@@ -573,8 +573,6 @@ endif ()
 # This configures: linking to Python::Module + pybind, output pre-/suf-fix, -fvisibility=hidden, strip release build
 pybind11_add_module(arcticdb_ext MODULE
         python/python_module.cpp
-        python/arctic_version.cpp
-        python/arctic_version.hpp
         python/reader.hpp
         python/adapt_read_dataframe.hpp
         )

cpp/arcticdb/python/python_module.cpp

@@ -24,6 +24,7 @@
 #include <arcticdb/util/global_lifetimes.hpp>
 #include <arcticdb/util/configs_map.hpp>
 #include <arcticdb/util/error_code.hpp>
+#include <arcticdb/util/version_number.hpp>
 
 #include <pybind11/pybind11.h>
 #include <folly/system/ThreadName.h>
@@ -290,10 +291,6 @@ PYBIND11_MODULE(arcticdb_ext, m) {
 
     register_termination_handler();
 
-#ifdef VERSION_INFO
-    m.attr("__version__") = VERSION_INFO;
-#else
-    m.attr("__version__") = "dev";
-#endif
+    m.attr("__version__") = ARCTICDB_VERSION_STR;
 }
 

cpp/arcticdb/util/global_lifetimes.cpp

@@ -13,6 +13,7 @@
 #include <arcticdb/log/log.hpp>
 #include <arcticdb/entity/metrics.hpp>
 #include <arcticdb/util/buffer_pool.hpp>
+#include <arcticdb/util/writer_info.hpp>
 
 #if defined(_MSC_VER) && defined(_DEBUG)
 #include <crtdbg.h>

cpp/arcticdb/util/version_number.hpp

@@ -0,0 +1,17 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file LICENSE.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+#ifndef ARCTICDB_MAJOR
+
+// The lines below until "END" are updated mechanically:
+#define ARCTICDB_MAJOR 1
+#define ARCTICDB_MINOR 2
+#define ARCTICDB_PATCH 0
+#define ARCTICDB_VERSION_STR "1.2.0-rc1"
+// END <- DO NOT MODIFY
+
+#define ARCTICDB_VERSION_INT ((ARCTICDB_MAJOR * 1000U + ARCTICDB_MINOR) * 1000U + ARCTICDB_PATCH)
+#endif

cpp/arcticdb/util/writer_info.hpp

@@ -0,0 +1,44 @@
+/* Copyright 2023 Man Group Operations Limited
+ *
+ * Use of this software is governed by the Business Source License 1.1 included in the file LICENSE.txt.
+ *
+ * As of the Change Date specified in that file, in accordance with the Business Source License, use of this software will be governed by the Apache License, version 2.0.
+ */
+#pragma once
+
+#include <arcticdb/entity/protobufs.hpp>
+#include "version_number.hpp"
+
+namespace arcticdb::util {
+
+inline void set_static_writer_info_fields(proto::descriptors::WriterInfo& proto) {
+    using Proto = proto::descriptors::WriterInfo;
+
+#if defined(__linux__)
+    proto.set_platform(Proto::LINUX);
+#elif defined(_WIN32)
+    proto.set_platform(Proto::WINDOWS);
+#elif defined(__APPLE__)
+    proto.set_platform(Proto::APPLE);
+#else
+#warning "Please add your OS to the protobuf"
+    proto.set_platform(Proto::UNKNOWN_PLATFORM);
+#endif
+
+#if defined(__clang_major__)
+    proto.set_cc(Proto::CLANG);
+    proto.set_cc_version((__clang_major__ * 1000 + __clang_minor__) * 1000 + __clang_patchlevel__);
+#elif defined(__GNUC__)
+    proto.set_cc(Proto::GCC);
+    proto.set_cc_version((__GNUC__ * 1000 + __GNUC_MINOR__) * 1000 + __GNUC_PATCHLEVEL__);
+#elif defined(_MSC_VER)
+    proto.set_cc(Proto::MSVC);
+    proto.set_cc_version(_MSC_FULL_VER);
+#else
+#warning "Please add your compiler to the protobuf"
+    proto.set_cc(Proto::UNKNOWN_CC);
+#endif
+
+    proto.set_version(ARCTICDB_VERSION_INT);
+}
+}

cpp/arcticdb/version/python_bindings.cpp

@@ -11,7 +11,7 @@
 #include <pybind11/numpy.h>
 #include <pybind11/operators.h>
 #include <arcticdb/version/version_store_api.hpp>
-#include <arcticdb/python/arctic_version.hpp>
+#include <arcticdb/util/version_number.hpp>
 #include <arcticdb/python/python_utils.hpp>
 #include <arcticdb/pipeline/column_stats.hpp>
 #include <arcticdb/pipeline/query.hpp>
@@ -21,6 +21,7 @@
 #include <arcticdb/processing/execution_context.hpp>
 #include <arcticdb/pipeline/value_set.hpp>
 #include <arcticdb/python/adapt_read_dataframe.hpp>
+#include <arcticdb/python/python_utils.hpp>
 #include <arcticdb/version/schema_checks.hpp>
 
 namespace arcticdb::version_store {
@@ -579,9 +580,14 @@ void register_bindings(py::module &m, py::exception<arcticdb::ArcticException>&
         .def("latest_timestamp",
              &PythonVersionStore::latest_timestamp,
              "Returns latest timestamp of a symbol")
+        .def("get_latest_symbol_list_compaction_descriptor",
+            [](PythonVersionStore& v) {
+                auto maybe_descriptor = v.get_latest_symbol_list_compaction_descriptor();
+                return maybe_descriptor ? python_util::pb_to_python(*maybe_descriptor) : py::none();
+            })
         ;
 
-    m.def("get_version_string", &get_arcticdb_version_string);
+    m.def("get_version_string", []() {return ARCTICDB_VERSION_STR; }, "Deprecated. Use arcticdb.__version__");
 
     m.def("read_runtime_config", [](const py::object object) {
         auto config = RuntimeConfig{};

cpp/arcticdb/version/symbol_list.cpp

@@ -6,6 +6,7 @@
  */
 
 #include <version/symbol_list.hpp>
+#include <arcticdb/util/writer_info.hpp>
 
 namespace arcticdb {
 
@@ -17,6 +18,7 @@ struct SymbolList::LoadResult {
     mutable KeyVector symbol_list_keys;
     /** The last CompactionId key in symbol_list_keys, if any. */
     std::optional<KeyVectorItr> maybe_last_compaction;
+    SymbolList::OptDescriptor maybe_last_compaction_descriptor;
     mutable CollectionType symbols;
     arcticdb::timestamp timestamp;
 
@@ -36,7 +38,8 @@ SymbolList::LoadResult SymbolList::attempt_load(const std::shared_ptr<Store>& st
     if (load_result.maybe_last_compaction) {
         load_result.timestamp = load_result.symbol_list_keys.rbegin()->creation_ts(); // Per step 5
         load_result.symbols = load_from_symbol_list_keys(store,
-                {*load_result.maybe_last_compaction, load_result.symbol_list_keys.cend()});
+                {*load_result.maybe_last_compaction, load_result.symbol_list_keys.cend()},
+                load_result.maybe_last_compaction_descriptor);
     } else {
         load_result.timestamp = store->current_timestamp();
         load_result.symbols = load_from_version_keys(store);
@@ -61,7 +64,8 @@ SymbolList::CollectionType SymbolList::load(const std::shared_ptr<Store>& store,
                 SYMBOL_LIST_RUNTIME_LOG("Checking whether we still need to compact under lock");
                 if (can_update_symbol_list(store, load_result.maybe_last_compaction)) {
                     // Step 5
-                    auto written = write_symbols(store, load_result.symbols, compaction_id, load_result.timestamp).get();
+                    auto written = write_symbols(store, load_result.symbols, compaction_id, load_result.timestamp,
+                            load_result.maybe_last_compaction_descriptor).get();
                     delete_keys(store, load_result.detach_symbol_list_keys(), std::get<AtomKey>(written)).get();
                 }
             } else {
@@ -136,11 +140,11 @@ bool SymbolList::can_update_symbol_list(const std::shared_ptr<Store>& store,
         return symbols;
     }
 
-    folly::Future<VariantKey> SymbolList::write_symbols(
-        const std::shared_ptr<Store>& store,
-        const CollectionType& symbols,
-        const StreamId& stream_id,
-        timestamp creation_ts)  {
+    folly::Future<VariantKey> SymbolList::write_symbols(const std::shared_ptr<Store>& store,
+            const CollectionType& symbols,
+            const StreamId& stream_id,
+            timestamp creation_ts,
+            const OptDescriptor& existing_descriptor) {
 
         SYMBOL_LIST_RUNTIME_LOG("Writing {} symbols to symbol list cache", symbols.size());
         SegmentInMemory list_segment{symbol_stream_descriptor(stream_id)};
@@ -159,25 +163,34 @@ bool SymbolList::can_update_symbol_list(const std::shared_ptr<Store>& store,
             );
             list_segment.end_row();
         }
-        if(symbols.empty()) {
-            google::protobuf::Any any = {};
-            arcticdb::proto::descriptors::SymbolListDescriptor metadata;
+
+        arcticdb::proto::descriptors::SymbolListDescriptor metadata;
+        if (existing_descriptor) {
+            metadata.CopyFrom(*existing_descriptor);
+            if (ARCTICDB_VERSION_INT < metadata.min_writer_version()) {
+                metadata.set_min_writer_version(ARCTICDB_VERSION_INT);
+            }
+        } else {
             metadata.set_enabled(true);
-            any.PackFrom(metadata);
-            list_segment.set_metadata(std::move(any));
+            util::set_static_writer_info_fields(*metadata.mutable_initial_writer());
+            metadata.set_min_writer_version(ARCTICDB_VERSION_INT);
         }
+        google::protobuf::Any any = {};
+        any.PackFrom(metadata);
+        list_segment.set_metadata(std::move(any));
+
         return store->write(KeyType::SYMBOL_LIST, 0, stream_id, creation_ts, 0, 0, std::move(list_segment));
     }
 
-    SymbolList::CollectionType SymbolList::load_from_symbol_list_keys(
-            const std::shared_ptr<StreamSource>& store,
-            const folly::Range<KeyVectorItr>& keys) {
+    SymbolList::CollectionType SymbolList::load_from_symbol_list_keys(const std::shared_ptr<StreamSource>& store,
+            const folly::Range<KeyVectorItr>& keys,
+            OptDescriptor& descriptor_out) {
         SYMBOL_LIST_RUNTIME_LOG("Loading symbols from symbol list keys");
         bool read_compaction = false;
         CollectionType symbols{};
         for(const auto& key : keys) {
             if(key.id() == compaction_id) {
-                read_list_from_storage(store, key, symbols);
+                read_list_from_storage(store, key, symbols, descriptor_out);
                 read_compaction = true;
             }
             else {
@@ -200,10 +213,17 @@ bool SymbolList::can_update_symbol_list(const std::shared_ptr<Store>& store,
         return symbols;
     }
 
-    void SymbolList::read_list_from_storage(const std::shared_ptr<StreamSource>& store, const AtomKey& key,
-                                            CollectionType& symbols) {
+    void SymbolList::read_list_from_storage(const std::shared_ptr<StreamSource>& store,
+            const AtomKey& key,
+            CollectionType& symbols,
+            OptDescriptor& descriptor_out) {
         ARCTICDB_DEBUG(log::version(), "Reading list from storage with key {}", key);
         auto key_seg = store->read(key).get().second;
+        if (key_seg.metadata()) {
+            descriptor_out.emplace();
+            key_seg.metadata()->UnpackTo(&descriptor_out.value());
+        }
+
         auto field_desc = key_seg.descriptor().field_at(0);
         missing_data::check<ErrorCode::E_UNREADABLE_SYMBOL_LIST>(field_desc.has_value(),
             "Expected at least one column in symbol list with key {}", key);
@@ -268,4 +288,14 @@ bool SymbolList::can_update_symbol_list(const std::shared_ptr<Store>& store,
         }
     }
 
+SymbolList::OptDescriptor SymbolList::get_latest_compaction_descriptor(const std::shared_ptr<Store>& store) {
+    auto result = attempt_load(store);
+    if (result.maybe_last_compaction) {
+        auto any = store->read_metadata(*result.maybe_last_compaction.value()).get().second;
+        proto::descriptors::SymbolListDescriptor metadata;
+        any->UnpackTo(&metadata);
+        return metadata;
+    }
+    return {};
+}
 } //namespace arcticdb

cpp/arcticdb/version/symbol_list.hpp

@@ -85,6 +85,10 @@ class SymbolList {
         delete_all_keys_of_type(KeyType::SYMBOL_LIST, store, true);
     }
 
+    using OptDescriptor = std::optional<proto::descriptors::SymbolListDescriptor>;
+    // For diagnostics.
+    OptDescriptor get_latest_compaction_descriptor(const std::shared_ptr<Store>& store);
+
 private:
     struct LoadResult;
 
@@ -106,18 +110,20 @@ class SymbolList {
 
     [[nodiscard]] CollectionType load_from_version_keys(const std::shared_ptr<Store>& store);
 
-    [[nodiscard]] folly::Future<VariantKey> write_symbols(
-            const std::shared_ptr<Store>& store,
+    [[nodiscard]] folly::Future<VariantKey> write_symbols(const std::shared_ptr<Store>& store,
             const CollectionType& symbols,
             const StreamId& stream_id,
-            timestamp creation_ts);
+            timestamp creation_ts,
+            const OptDescriptor& existing_descriptor);
 
-    [[nodiscard]] CollectionType load_from_symbol_list_keys(
-            const std::shared_ptr<StreamSource>& store,
-            const folly::Range<KeyVectorItr>& keys);
+    [[nodiscard]] SymbolList::CollectionType load_from_symbol_list_keys(const std::shared_ptr<StreamSource>& store,
+            const folly::Range<KeyVectorItr>& keys,
+            OptDescriptor& descriptor_out);
 
-    void read_list_from_storage(const std::shared_ptr<StreamSource>& store, const AtomKey& key,
-            CollectionType& symbols);
+    void read_list_from_storage(const std::shared_ptr<StreamSource>& store,
+            const AtomKey& key,
+            CollectionType& symbols,
+            OptDescriptor& descriptor_out);
 
     [[nodiscard]] KeyVector get_all_symbol_list_keys(const std::shared_ptr<StreamSource>& store) const;
 

cpp/arcticdb/version/version_store_api.hpp

@@ -314,6 +314,11 @@ class PythonVersionStore : public LocalVersionedEngine {
 
     std::vector<AtomKey> get_version_history(const StreamId& stream_id);
 
+    // For diagnostics.
+    auto get_latest_symbol_list_compaction_descriptor() {
+        return symbol_list_ptr()->get_latest_compaction_descriptor(store());
+    }
+
 private:
 
     std::vector<VersionedItem> batch_write_index_keys_to_version_map(