feat: ETLng Registry (#1713)

For #1597

src/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_subdirectory(util)
 add_subdirectory(data)
 add_subdirectory(etl)
+add_subdirectory(etlng)
 add_subdirectory(feed)
 add_subdirectory(rpc)
 add_subdirectory(web)

src/app/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_library(clio_app)
 target_sources(clio_app PRIVATE CliArgs.cpp ClioApplication.cpp)
 
-target_link_libraries(clio_app PUBLIC clio_etl clio_feed clio_web clio_rpc)
+target_link_libraries(clio_app PUBLIC clio_etl clio_etlng clio_feed clio_web clio_rpc)

src/etlng/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_library(clio_etlng INTERFACE)
+
+# target_sources(clio_etlng PRIVATE )
+
+target_link_libraries(clio_etlng INTERFACE clio_data)

src/etlng/Models.hpp

@@ -0,0 +1,129 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "util/Concepts.hpp"
+
+#include <boost/json/object.hpp>
+#include <fmt/core.h>
+#include <xrpl/basics/Blob.h>
+#include <xrpl/basics/base_uint.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/get_ledger.pb.h>
+#include <xrpl/proto/org/xrpl/rpc/v1/ledger.pb.h>
+#include <xrpl/protocol/LedgerHeader.h>
+#include <xrpl/protocol/STTx.h>
+#include <xrpl/protocol/TxFormats.h>
+#include <xrpl/protocol/TxMeta.h>
+
+#include <cstdint>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace etlng::model {
+
+/**
+ * @brief A specification for the Registry.
+ *
+ * This specification simply defines the transaction types that are to be filtered out from the incoming transactions by
+ * the Registry for its `onTransaction` and `onInitialTransaction` hooks.
+ * It's a compilation error to list the same transaction type more than once.
+ */
+template <ripple::TxType... Types>
+    requires(util::hasNoDuplicates(Types...))
+struct Spec {
+    static constexpr bool SpecTag = true;
+
+    /**
+     * @brief Checks if the transaction type was requested.
+     *
+     * @param type The transaction type
+     * @return true if the transaction was requested; false otherwise
+     */
+    [[nodiscard]] constexpr static bool
+    wants(ripple::TxType type) noexcept
+    {
+        return ((Types == type) || ...);
+    }
+};
+
+/**
+ * @brief Represents a single transaction on the ledger.
+ */
+struct Transaction {
+    std::string raw;  // raw binary blob
+    std::string metaRaw;
+
+    // unpacked blob and meta
+    ripple::STTx sttx;
+    ripple::TxMeta meta;
+
+    // commonly used stuff
+    ripple::uint256 id;
+    std::string key;  // key is the above id as a string of 32 characters
+    ripple::TxType type;
+};
+
+/**
+ * @brief Represents a single object on the ledger.
+ */
+struct Object {
+    /**
+     * @brief Modification type for the object.
+     */
+    enum class ModType : int {
+        Unspecified = 0,
+        Created = 1,
+        Modified = 2,
+        Deleted = 3,
+    };
+
+    ripple::uint256 key;
+    std::string keyRaw;
+    ripple::Blob data;
+    std::string dataRaw;
+    std::string successor;
+    std::string predecessor;
+
+    ModType type;
+};
+
+/**
+ * @brief Represents a book successor.
+ */
+struct BookSuccessor {
+    std::string firstBook;
+    std::string bookBase;
+};
+
+/**
+ * @brief Represents an entire ledger diff worth of transactions and objects.
+ */
+struct LedgerData {
+    std::vector<Transaction> transactions;
+    std::vector<Object> objects;
+    std::optional<std::vector<BookSuccessor>> successors;
+
+    ripple::LedgerHeader header;
+    std::string rawHeader;
+    uint32_t seq;
+};
+
+}  // namespace etlng::model

src/etlng/RegistryInterface.hpp

@@ -0,0 +1,108 @@
+//------------------------------------------------------------------------------
+/*
+    This file is part of clio: https://github.com/XRPLF/clio
+    Copyright (c) 2024, the clio developers.
+
+    Permission to use, copy, modify, and distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE  SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH  REGARD  TO  THIS  SOFTWARE  INCLUDING  ALL  IMPLIED  WARRANTIES  OF
+    MERCHANTABILITY  AND  FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY  SPECIAL,  DIRECT,  INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER  RESULTING  FROM  LOSS  OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION  OF  CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+//==============================================================================
+
+#pragma once
+
+#include "etlng/Models.hpp"
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace etlng {
+
+/**
+ * @brief The interface for a registry that can dispatch transactions and objects to extensions.
+ *
+ * This class defines the interface for dispatching data through to extensions.
+ *
+ * @note
+ * The registry itself consists of Extensions.
+ * Each extension must define at least one valid hook:
+ * - for ongoing ETL dispatch:
+ *   - void onLedgerData(etlng::model::LedgerData const&)
+ *   - void onTransaction(uint32_t, etlng::model::Transaction const&)
+ *   - void onObject(uint32_t, etlng::model::Object const&)
+ * - for initial ledger load
+ *   - void onInitialData(etlng::model::LedgerData const&)
+ *   - void onInitialTransaction(uint32_t, etlng::model::Transaction const&)
+ * - for initial objects (called for each downloaded batch)
+ *   - void onInitialObjects(uint32_t, std::vector<etlng::model::Object> const&, std::string)
+ *   - void onInitialObject(uint32_t, etlng::model::Object const&)
+ *
+ * When the registry dispatches (initial)data or objects, each of the above hooks will be called in order on each
+ * registered extension.
+ * This means that the order of execution is from left to right (hooks) and top to bottom (registered extensions).
+ *
+ * If either `onTransaction` or `onInitialTransaction` are defined, the extension will have to additionally define a
+ * Specification. The specification lists transaction types to filter from the incoming data such that `onTransaction`
+ * and `onInitialTransaction` are only called for the transactions that are of interest for the given extension.
+ *
+ * The specification is setup like so:
+ * @code{.cpp}
+ * struct Ext {
+ *   using spec = etlng::model::Spec<
+ *     ripple::TxType::ttNFTOKEN_BURN,
+ *     ripple::TxType::ttNFTOKEN_ACCEPT_OFFER,
+ *     ripple::TxType::ttNFTOKEN_CREATE_OFFER,
+ *     ripple::TxType::ttNFTOKEN_CANCEL_OFFER,
+ *     ripple::TxType::ttNFTOKEN_MINT>;
+ *
+ *   static void
+ *   onInitialTransaction(uint32_t, etlng::model::Transaction const&);
+ * };
+ * @endcode
+ */
+struct RegistryInterface {
+    virtual ~RegistryInterface() = default;
+
+    /**
+     * @brief Dispatch initial objects.
+     *
+     * These objects are received during initial ledger load.
+     *
+     * @param seq The sequence
+     * @param data The objects to dispatch
+     * @param lastKey The predcessor of the first object in data if known; an empty string otherwise
+     */
+    virtual void
+    dispatchInitialObjects(uint32_t seq, std::vector<model::Object> const& data, std::string lastKey) = 0;
+
+    /**
+     * @brief Dispatch initial ledger data.
+     *
+     * The transactions, header and edge keys are received during initial ledger load.
+     *
+     * @param data The data to dispatch
+     */
+    virtual void
+    dispatchInitialData(model::LedgerData const& data) = 0;
+
+    /**
+     * @brief Dispatch an entire ledger diff.
+     *
+     * This is used to dispatch incoming diffs through the extensions.
+     *
+     * @param data The data to dispatch
+     */
+    virtual void
+    dispatch(model::LedgerData const& data) = 0;
+};
+
+}  // namespace etlng