From 23c09837cb992d958a83cafdc7d6b7bff3d1eb15 Mon Sep 17 00:00:00 2001 From: Seungbaek Hong Date: Tue, 30 Jul 2024 15:17:17 +0900 Subject: [PATCH] [Layer] add Weight Layer - This layer contains only weights for building tensor-level graph **Self-evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Seungbaek Hong --- api/ccapi/include/layer.h | 17 ++- api/nntrainer-api-common.h | 3 +- nntrainer/app_context.cpp | 3 + nntrainer/layers/common_properties.h | 10 ++ nntrainer/layers/meson.build | 1 + nntrainer/layers/weight_layer.cpp | 87 +++++++++++++++ nntrainer/layers/weight_layer.h | 104 ++++++++++++++++++ nntrainer/utils/base_properties.h | 13 +++ test/ccapi/unittest_ccapi.cpp | 3 + test/unittest/layers/meson.build | 1 + .../layers/unittest_layers_weight.cpp | 30 +++++ 11 files changed, 267 insertions(+), 5 deletions(-) create mode 100644 nntrainer/layers/weight_layer.cpp create mode 100644 nntrainer/layers/weight_layer.h create mode 100644 test/unittest/layers/unittest_layers_weight.cpp diff --git a/api/ccapi/include/layer.h b/api/ccapi/include/layer.h index e384231e6f..19266ae5a7 100644 --- a/api/ccapi/include/layer.h +++ b/api/ccapi/include/layer.h @@ -36,6 +36,7 @@ namespace train { */ enum LayerType { LAYER_IN = ML_TRAIN_LAYER_TYPE_INPUT, /**< Input Layer type */ + LAYER_WEIGHT = ML_TRAIN_LAYER_TYPE_WEIGHT, /**< Weight Layer type */ LAYER_FC = ML_TRAIN_LAYER_TYPE_FC, /**< Fully Connected Layer type */ LAYER_SWIGLU = ML_TRAIN_LAYER_TYPE_SWIGLU, /**< Swiglu Layer type */ LAYER_BN = ML_TRAIN_LAYER_TYPE_BN, /**< Batch Normalization Layer type */ @@ -102,7 +103,7 @@ enum LayerType { derivative */ LAYER_UPSAMPLE2D, /**< Upsample 2D Layer type */ LAYER_RMSNORM = ML_TRAIN_LAYER_TYPE_RMSNORM, /** &properties = {}) { return createLayer(LayerType::LAYER_IN, properties); } +/** + * @brief Helper function to create weight layer + */ +inline std::unique_ptr +WeightLayer(const std::vector &properties = {}) { + return createLayer(LayerType::LAYER_WEIGHT, properties); +} + /** * @brief Helper function to create fully connected layer */ @@ -311,9 +320,9 @@ Swiglu(const std::vector &properties = {}, /** * @brief Helper function to create RMS normalization layer for GPU */ -inline std::unique_ptr RMSNormCl( - const std::vector &properties = {}, - const LayerComputeEngine &compute_engine = LayerComputeEngine::GPU) { +inline std::unique_ptr +RMSNormCl(const std::vector &properties = {}, + const LayerComputeEngine &compute_engine = LayerComputeEngine::GPU) { return createLayer(LayerType::LAYER_RMSNORM, properties, compute_engine); } diff --git a/api/nntrainer-api-common.h b/api/nntrainer-api-common.h index 76d9976f3b..97a5a71fad 100644 --- a/api/nntrainer-api-common.h +++ b/api/nntrainer-api-common.h @@ -64,6 +64,7 @@ typedef enum { 28, /**< Positional Encoding Layer type (Since 7.0) */ ML_TRAIN_LAYER_TYPE_IDENTITY = 29, /**< Identity Layer type (Since 8.0) */ ML_TRAIN_LAYER_TYPE_SWIGLU = 30, /**< Swiglu Layer type */ + ML_TRAIN_LAYER_TYPE_WEIGHT = 31, /**< Weight Layer type (Since 9.0)*/ ML_TRAIN_LAYER_TYPE_PREPROCESS_FLIP = 300, /**< Preprocess flip Layer (Since 6.5) */ ML_TRAIN_LAYER_TYPE_PREPROCESS_TRANSLATE = @@ -77,7 +78,7 @@ typedef enum { ML_TRAIN_LAYER_TYPE_LOSS_CROSS_ENTROPY_SOFTMAX = 502, /**< Cross Entropy with Softmax Loss Layer type (Since 6.5) */ ML_TRAIN_LAYER_TYPE_RMSNORM = 503, /**< Cross Entropy with */ - ML_TRAIN_LAYER_TYPE_UNKNOWN = 999 /**< Unknown Layer */ + ML_TRAIN_LAYER_TYPE_UNKNOWN = 999 /**< Unknown Layer */ } ml_train_layer_type_e; /** diff --git a/nntrainer/app_context.cpp b/nntrainer/app_context.cpp index fad75f9ae0..09b6fd10f4 100644 --- a/nntrainer/app_context.cpp +++ b/nntrainer/app_context.cpp @@ -74,6 +74,7 @@ #include #include #include +#include #include #ifdef ENABLE_TFLITE_BACKBONE @@ -245,6 +246,8 @@ static void add_default_object(AppContext &ac) { using LayerType = ml::train::LayerType; ac.registerFactory(nntrainer::createLayer, InputLayer::type, LayerType::LAYER_IN); + ac.registerFactory(nntrainer::createLayer, WeightLayer::type, + LayerType::LAYER_WEIGHT); ac.registerFactory(nntrainer::createLayer, FullyConnectedLayer::type, LayerType::LAYER_FC); ac.registerFactory(nntrainer::createLayer, diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h index 2591ab454b..64818be4fc 100644 --- a/nntrainer/layers/common_properties.h +++ b/nntrainer/layers/common_properties.h @@ -114,6 +114,16 @@ class Trainable : public nntrainer::Property { using prop_tag = bool_prop_tag; }; +/** + * @brief Tensor Dimension property + * + */ +class TensorDimension : public TensorDimProperty { +public: + static constexpr const char *key = "dim"; /**< unique key to access */ + using prop_tag = dimension_prop_tag; /**< property type */ +}; + /** * @brief trainable property, use this to set and check how if certain layer is * trainable diff --git a/nntrainer/layers/meson.build b/nntrainer/layers/meson.build index 75a59605ab..c612d8c177 100644 --- a/nntrainer/layers/meson.build +++ b/nntrainer/layers/meson.build @@ -4,6 +4,7 @@ nntrainer_inc_abs += meson.current_source_dir() / 'loss' layer_sources = [ 'activation_layer.cpp', + 'weight_layer.cpp', 'addition_layer.cpp', 'attention_layer.cpp', 'mol_attention_layer.cpp', diff --git a/nntrainer/layers/weight_layer.cpp b/nntrainer/layers/weight_layer.cpp new file mode 100644 index 0000000000..840434891d --- /dev/null +++ b/nntrainer/layers/weight_layer.cpp @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 SeungBaek Hong + * + * @file weight_layer.cpp + * @date 2 August 2024 + * @brief This is a layer that simply stores a weight tensor without any + * operation. + * @see https://github.com/nnstreamer/nntrainer + * @author SeungBaek Hong + * @bug No known bugs except for NYI items + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace nntrainer { + +static constexpr size_t SINGLE_INOUT_IDX = 0; + +WeightLayer::WeightLayer() : LayerImpl() { + weight_idx.fill(std::numeric_limits::max()); +} + +void WeightLayer::finalize(InitLayerContext &context) { + auto &weight_regularizer = + std::get(*layer_impl_props); + auto &weight_regularizer_constant = + std::get(*layer_impl_props); + auto &weight_initializer = + std::get(*layer_impl_props); + auto &weight_decay = std::get(*layer_impl_props); + + const auto &weight_dim = std::get(weight_props).get(); + + std::vector output_dims(1); + + output_dims[0] = weight_dim; + + output_dims[0].setTensorType( + {context.getFormat(), context.getActivationDataType()}); + + context.setOutputDimensions(output_dims); + + weight_idx[0] = context.requestWeight( + weight_dim, weight_initializer, weight_regularizer, + weight_regularizer_constant, weight_decay, "weight", true); +} + +void WeightLayer::exportTo(Exporter &exporter, + const ml::train::ExportMethods &method) const { + LayerImpl::exportTo(exporter, method); + exporter.saveResult(weight_props, method, this); +} + +void WeightLayer::setProperty(const std::vector &values) { + auto remain_props = loadProperties(values, weight_props); + LayerImpl::setProperty(remain_props); +} + +void WeightLayer::forwarding(RunLayerContext &context, bool training) { + Tensor &weight = context.getWeight(weight_idx[0]); + Tensor &output = context.getOutput(SINGLE_INOUT_IDX); + output.copy(weight); +} + +void WeightLayer::calcDerivative(RunLayerContext &context) { + throw exception::not_supported( + "calcDerivative for weight layer is not supported"); +} + +void WeightLayer::calcGradient(RunLayerContext &context) { + Tensor &djdw = context.getWeightGrad(weight_idx[0]); + const Tensor &derivative_ = context.getIncomingDerivative(SINGLE_INOUT_IDX); + djdw.copy(derivative_); +} + +} /* namespace nntrainer */ diff --git a/nntrainer/layers/weight_layer.h b/nntrainer/layers/weight_layer.h new file mode 100644 index 0000000000..6c3a42ffc7 --- /dev/null +++ b/nntrainer/layers/weight_layer.h @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 SeungBaek Hong + * + * @file weight_layer.h + * @date 2 August 2024 + * @brief This is a layer that simply stores a weight tensor without any + * operation. + * @see https://github.com/nnstreamer/nntrainer + * @author SeungBaek Hong + * @bug No known bugs except for NYI items + * + */ + +#ifndef __WEIGHT_LAYER_H__ +#define __WEIGHT_LAYER_H__ +#ifdef __cplusplus + +#include +#include + +namespace nntrainer { + +/** + * @class Weight Layer + * @brief A layer that simply stores a weight tensor + */ +class WeightLayer : public LayerImpl { +public: + /** + * @brief Constructor of Weight Layer + */ + WeightLayer(); + + /** + * @brief Destructor of Weight Layer + */ + ~WeightLayer() = default; + + /** + * @brief Move constructor. + * @param[in] WeightLayer && + */ + WeightLayer(WeightLayer &&rhs) noexcept = default; + + /** + * @brief Move assignment operator. + * @parma[in] rhs WeightLayer to be moved. + */ + WeightLayer &operator=(WeightLayer &&rhs) = default; + + /** + * @copydoc Layer::finalize(InitLayerContext &context) + */ + void finalize(InitLayerContext &context) override; + + /** + * @copydoc Layer::forwarding(RunLayerContext &context, bool training) + */ + void forwarding(RunLayerContext &context, bool training) override; + + /** + * @copydoc Layer::calcDerivative(RunLayerContext &context) + */ + void calcDerivative(RunLayerContext &context) override; + + /** + * @copydoc Layer::calcGradient(RunLayerContext &context) + */ + void calcGradient(RunLayerContext &context) override; + + /** + * @copydoc Layer::exportTo(Exporter &exporter, ml::train::ExportMethods + * method) + */ + void exportTo(Exporter &exporter, + const ml::train::ExportMethods &method) const override; + + /** + * @copydoc Layer::getType() + */ + const std::string getType() const override { return WeightLayer::type; }; + + /** + * @copydoc Layer::supportBackwarding() + */ + bool supportBackwarding() const override { return true; } + + /** + * @copydoc Layer::setProperty(const PropertyType type, const std::string + * &value) + */ + void setProperty(const std::vector &values) override; + + inline static const std::string type = "weight"; + +private: + std::tuple weight_props; + std::array weight_idx; /**< indices of the weights */ +}; +} // namespace nntrainer + +#endif /* __cplusplus */ +#endif /* __WEIGHT_LAYER_H__ */ diff --git a/nntrainer/utils/base_properties.h b/nntrainer/utils/base_properties.h index 259637a6d9..c24948a33b 100644 --- a/nntrainer/utils/base_properties.h +++ b/nntrainer/utils/base_properties.h @@ -277,6 +277,19 @@ class EnumProperty : public Property { static EnumInfo enum_info_; }; +/** + * @brief abstract class for tensor dimension + * + */ +class TensorDimProperty : public Property { +public: + /** + * @brief Destroy the TensorDim Property object + * + */ + virtual ~TensorDimProperty() = default; +}; + /** * @brief abstract class for positive integer * diff --git a/test/ccapi/unittest_ccapi.cpp b/test/ccapi/unittest_ccapi.cpp index 98c006430d..34c99f4f5b 100644 --- a/test/ccapi/unittest_ccapi.cpp +++ b/test/ccapi/unittest_ccapi.cpp @@ -61,6 +61,9 @@ TEST(ccapi_layer, construct_02_p) { EXPECT_NO_THROW(layer = ml::train::layer::Input()); EXPECT_EQ(layer->getType(), "input"); + EXPECT_NO_THROW(layer = ml::train::layer::WeightLayer()); + EXPECT_EQ(layer->getType(), "weight"); + EXPECT_NO_THROW(layer = ml::train::layer::FullyConnected()); EXPECT_EQ(layer->getType(), "fully_connected"); diff --git a/test/unittest/layers/meson.build b/test/unittest/layers/meson.build index 59992b17f6..c65609e881 100644 --- a/test/unittest/layers/meson.build +++ b/test/unittest/layers/meson.build @@ -36,6 +36,7 @@ test_target = [ 'unittest_layers.cpp', 'unittest_layers_impl.cpp', 'unittest_layers_input.cpp', + 'unittest_layers_weight.cpp', 'unittest_layers_loss.cpp', 'unittest_layers_fully_connected.cpp', 'unittest_layers_batch_normalization.cpp', diff --git a/test/unittest/layers/unittest_layers_weight.cpp b/test/unittest/layers/unittest_layers_weight.cpp new file mode 100644 index 0000000000..3bc6c8cced --- /dev/null +++ b/test/unittest/layers/unittest_layers_weight.cpp @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 +/** + * Copyright (C) 2024 SeungBaek Hong + * + * @file unittest_layers_weight.cpp + * @date 30 July 2024 + * @brief Weight Layer Test + * @see https://github.com/nnstreamer/nntrainer + * @author SeungBaek Hong + * @bug No known bugs except for NYI items + */ +#include + +#include + +#include +#include + +auto semantic_weight = LayerSemanticsParamType( + nntrainer::createLayer, nntrainer::WeightLayer::type, + {"dim=1:1:1"}, LayerCreateSetPropertyOptions::AVAILABLE_FROM_APP_CONTEXT, + false, 1); + +auto semantic_weight_multi = LayerSemanticsParamType( + nntrainer::createLayer, nntrainer::WeightLayer::type, + {"dim=1:1:1"}, LayerCreateSetPropertyOptions::AVAILABLE_FROM_APP_CONTEXT, + false, 2); + +GTEST_PARAMETER_TEST(Weight, LayerSemantics, + ::testing::Values(semantic_weight, semantic_weight_multi));