From 9a3c54788759d42fe6ac07d5ab51271038887b76 Mon Sep 17 00:00:00 2001 From: Eunju Yang Date: Fri, 27 Dec 2024 14:50:40 +0900 Subject: [PATCH] [ Subgraph ] Feat/ add subgraph_realizer - This commit adds subgraph_realizer. - The subgraph_realizer supports to create multiple subgraphs with is_shared_subgraph property. **Self-evaluation:** Build test: [X]Passed [ ]Failed [ ]Skipped Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Eunju Yang --- api/ccapi/include/model.h | 7 +- nntrainer/compiler/meson.build | 1 + nntrainer/compiler/subgraph_realizer.cpp | 82 ++++++++++++++++++ nntrainer/compiler/subgraph_realizer.h | 103 +++++++++++++++++++++++ nntrainer/models/neuralnet.cpp | 19 ++++- 5 files changed, 208 insertions(+), 4 deletions(-) create mode 100644 nntrainer/compiler/subgraph_realizer.cpp create mode 100644 nntrainer/compiler/subgraph_realizer.h diff --git a/api/ccapi/include/model.h b/api/ccapi/include/model.h index a9e02e7333..a3723ab1a0 100644 --- a/api/ccapi/include/model.h +++ b/api/ccapi/include/model.h @@ -75,6 +75,7 @@ enum class ModelType { enum class ReferenceLayersType { BACKBONE, /** backbone */ RECURRENT, /** recurrent */ + SUBGRAPH, /** subgraph */ }; /** @@ -308,7 +309,8 @@ class Model { * @param[in] init_seq_len initial sequence length * @param[in] from current working step index * @param[in] to next working step index - * @param[in] output_hidden_state return last hidden state if true else return all hidden state + * @param[in] output_hidden_state return last hidden state if true else return + * all hidden state * @retval list of output as float * * @note The output memory must not be freed by the caller */ @@ -316,8 +318,7 @@ class Model { incremental_inference(unsigned int batch, const std::vector &input, const std::vector &label, unsigned int init_seq_len, unsigned int from, - unsigned int to, - bool output_hidden_state = false) = 0; + unsigned int to, bool output_hidden_state = false) = 0; /** * @brief Summarize the model diff --git a/nntrainer/compiler/meson.build b/nntrainer/compiler/meson.build index cd0fe644d3..530527d14f 100644 --- a/nntrainer/compiler/meson.build +++ b/nntrainer/compiler/meson.build @@ -10,6 +10,7 @@ compiler_sources = [ 'multiout_realizer.cpp', 'bn_realizer.cpp', 'loss_realizer.cpp', + 'subgraph_realizer.cpp', ] compiler_headers = [ diff --git a/nntrainer/compiler/subgraph_realizer.cpp b/nntrainer/compiler/subgraph_realizer.cpp new file mode 100644 index 0000000000..aa9a83b223 --- /dev/null +++ b/nntrainer/compiler/subgraph_realizer.cpp @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 Eunju Yang + * + * @file subgraph_realizer.cpp + * @date 27 Dec 2024 + * @brief NNTrainer subgraph realizer + * @see https://github.com/nnstreamer/nntrainer + * @author Eunju Yang + * @bug No known bugs except for NYI items + */ + +#include +#include + +#include +#include +#include +#include +#include + +#include + +namespace nntrainer { + +SubgraphRealizer::SubgraphRealizer(const std::string scope, + const std::vector &properties, + const std::vector &input_conns) : + realizer_props(props::SubgraphIdx(), props::IsSharedSubgraph()), + input_conns(input_conns), + scope_name(scope), + subgraph_idx(0), + is_shared_subgraph(true) { + + auto left = loadProperties(properties, realizer_props); + NNTR_THROW_IF(!left.empty(), std::invalid_argument) + << "There is unparsed properties"; + + subgraph_idx = std::get(realizer_props).get(); + is_shared_subgraph = std::get(realizer_props).get(); + scope_full_name = scope + "/" + std::to_string(subgraph_idx); +} + +SubgraphRealizer::~SubgraphRealizer() {} + +/** + * @note + * subgraphrealize conducts the following two steps: + * 1. rename all the nodes in the subgraph + * to be unique by adding scope as its prefix + * 2. set shared_from property of each node + * in the subgraph to point to the original node + */ +GraphRepresentation +SubgraphRealizer::realize(const GraphRepresentation &reference) { + + auto subgraph_realizer = [this](const GraphRepresentation &reference_) { + RemapRealizer rename_mapper([this](std::string &name) { + for (auto &i : input_conns) { + if (i.getName() == name) { + return; + } + } + std::string scoped_name = scope_full_name + "/" + name; + subgraph_node_names[scoped_name] = name; + name = scoped_name; + }); + + auto nodes = rename_mapper.realize(reference_); + + if (is_shared_subgraph) { + for (auto &node : nodes) + node->setProperty({"shared_from=" + scope_name + "/0/" + + subgraph_node_names[node->getName()]}); + } + return nodes; + }; + + return subgraph_realizer(reference); +} + +} // namespace nntrainer \ No newline at end of file diff --git a/nntrainer/compiler/subgraph_realizer.h b/nntrainer/compiler/subgraph_realizer.h new file mode 100644 index 0000000000..9b56ff9da8 --- /dev/null +++ b/nntrainer/compiler/subgraph_realizer.h @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 Eunju Yang + * + * @file subgraph_realizer.h + * @date 27 Dec 2024 + * @brief NNTrainer subgraph realizer + * @see https://github.com/nnstreamer/nntrainer + * @author Eunju Yang + * @bug No known bugs except for NYI items + */ + +#ifndef __SUBGRAPH_REALIZER_H__ +#define __SUBGRAPH_REALIZER_H__ + +#include + +#include +#include +#include +#include + +#include + +namespace nntrainer { + +namespace props { +/** + * @brief Property subgraph_idx + */ +class SubgraphIdx final : public nntrainer::Property { +public: + SubgraphIdx(const unsigned int &val = 0) : + nntrainer::Property(val) { + set(val); + } + static constexpr const char *key = "subgraph_idx"; + using prop_tag = uint_prop_tag; +}; + +/** + * @brief Property is_shared_subgraph + */ +class IsSharedSubgraph final : public nntrainer::Property { +public: + IsSharedSubgraph(bool val = true) : nntrainer::Property(val) {} + static constexpr const char *key = "is_shared_subgraph"; + using prop_tag = bool_prop_tag; +}; +} // namespace props + +/** + * @brief SubGraph Realizer which adding some properties for subgraph + * construction. + * @param properties + * subgraph_idx = + * is_shared_subgraph = + */ +class SubgraphRealizer : public GraphRealizer { +public: + /** + * @brief Construct a new Subgraph Realizer object + * @note + * SubGraphRealizer do the two tasks: + * 1. Update name of the every node in subgraph with scope/ + * 2. The scope name can be varied according to its subgraph index, i.e., + * scope/idx/name + * 3. If is_shared_subgraph is true, then the scope name will be shared among + * subgraphs. + * @param properties + * subgraph_idx = + * is_shared_subgraph = + * @param input_conns input conns from outer side + */ + SubgraphRealizer(const std::string scope, + const std::vector &properties, + const std::vector &input_conns); + + /** + * @brief Destroy the subgraph realizer object + */ + ~SubgraphRealizer(); + + /** + * @brief realized graph + */ + GraphRepresentation realize(const GraphRepresentation &reference) override; + +private: + std::tuple + realizer_props; /**< subgraph properties */ + std::vector input_conns; + std::string scope_name; /**< scope name */ + std::string scope_full_name; + std::unordered_map + subgraph_node_names; /**< subgraph_name, original name */ + unsigned int subgraph_idx; + bool is_shared_subgraph; +}; + +} /* namespace nntrainer */ + +#endif /* __SUBGRAPH_REALIZER_H__ */ diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp index 9ed65bf0a9..e3a89f060f 100644 --- a/nntrainer/models/neuralnet.cpp +++ b/nntrainer/models/neuralnet.cpp @@ -49,6 +49,7 @@ #include #include #include +#include #include #ifdef ENABLE_TFLITE_INTERPRETER @@ -1459,7 +1460,23 @@ void NeuralNetwork::addWithReferenceLayers( new RecurrentRealizer(type_properties, input_conns, end_conns)); } - if (!scope.empty()) { + /** + * @brief add SubGraphRealizer when type is SUBGRAPH + * subgraphrealizer updates the name of the node. + * Thus, remaprealizer should not be applied again. + * @note scope name should be set to apply subgraph realizer. + */ + if (type == ml::train::ReferenceLayersType::SUBGRAPH) { + + // Applied only when `scope` is specified. + if (scope.empty()) { + ml_logd( + "The subgraph's scope name is not set. Subgraph setting is skipped."); + return; + } + realizers.emplace_back( + new SubgraphRealizer(scope, type_properties, input_conns)); + } else if (!scope.empty()) { realizers.emplace_back( new RemapRealizer([&scope, &input_conns](std::string &name) { for (auto &i : input_conns) {