Merge branch 'main' into new_add_convTranspose

Applications/LLaMA/jni/transpose_layer.h

@@ -58,7 +58,7 @@ class TransposeLayer final : public nntrainer::Layer {
   /**
    * @copydoc bool supportBackwarding() const
    */
-  bool supportBackwarding() const override { return true; };
+  bool supportBackwarding() const override { return false; };
 
   /**
    * @copydoc Layer::exportTo(Exporter &exporter, ExportMethods method)

api/ccapi/include/layer.h

@@ -107,8 +107,9 @@ enum LayerType {
   LAYER_LOSS_CONSTANT_DERIVATIVE, /**< Synthetic loss layer to feed constant
                                      derivative */
   LAYER_UPSAMPLE2D,               /**< Upsample 2D Layer type */
-  LAYER_RMSNORM = ML_TRAIN_LAYER_TYPE_RMSNORM, /**<RMS NORM Layer */
-  LAYER_UNKNOWN = ML_TRAIN_LAYER_TYPE_UNKNOWN  /**< Unknown */
+  LAYER_RMSNORM = ML_TRAIN_LAYER_TYPE_RMSNORM,     /**<RMS NORM Layer */
+  LAYER_TRANSPOSE = ML_TRAIN_LAYER_TYPE_TRANSPOSE, /**< Transpose Layer type */
+  LAYER_UNKNOWN = ML_TRAIN_LAYER_TYPE_UNKNOWN      /**< Unknown */
 };
 
 /**
@@ -136,11 +137,6 @@ class Layer {
    */
   virtual const std::string getType() const = 0;
 
-  /**
-   * @brief Initialize layer
-   */
-  virtual void initialize() = 0;
-
   /**
    * @brief     Default allowed properties
    * - input shape : string
@@ -369,6 +365,15 @@ RMSNormCl(const std::vector<std::string> &properties = {},
   return createLayer(LayerType::LAYER_RMSNORM, properties, compute_engine);
 }
 
+/**
+ * @brief Helper function to create Transpose layer
+ */
+inline std::unique_ptr<Layer>
+Transpose(const std::vector<std::string> &properties = {},
+          const LayerComputeEngine &compute_engine = LayerComputeEngine::CPU) {
+  return createLayer(LayerType::LAYER_TRANSPOSE, properties, compute_engine);
+}
+
 /**
  * @brief Helper function to create batch normalization layer
  */

api/nntrainer-api-common.h

@@ -69,7 +69,8 @@ typedef enum {
   ML_TRAIN_LAYER_TYPE_SUBTRACT = 33, /**< Subtract Layer type (Since 9.0)*/
   ML_TRAIN_LAYER_TYPE_MULTIPLY = 34, /**< Multiply Layer type (Since 9.0)*/
   ML_TRAIN_LAYER_TYPE_DIVIDE = 35,   /**< Divide Layer type (Since 9.0)*/
-  ML_TRAIN_LAYER_TYPE_CONV2D_TRANSPOSE = 36, /**< Convolution 2D Transpose Layer type (Since 9.0) */
+  ML_TRAIN_LAYER_TYPE_TRANSPOSE = 36, /**< Transpose Layer type */
+  ML_TRAIN_LAYER_TYPE_CONV2D_TRANSPOSE = 37, /**< Convolution 2D Transpose Layer type (Since 9.0) */
   ML_TRAIN_LAYER_TYPE_PREPROCESS_FLIP =
     300, /**< Preprocess flip Layer (Since 6.5) */
   ML_TRAIN_LAYER_TYPE_PREPROCESS_TRANSLATE =

docs/how-to-use-testcases.md

@@ -16,3 +16,19 @@ $ cd build
 $ ninja test
 ...
 ```
+
+### run test cases on Android
+
+- In order to run unittest on android, please set the environment following [[docs] how-to-run-example-android.md](how-to-run-example-android.md).
+- Then, you can run the unittest on Android as follows:
+
+```
+(nntrainer) $ ./tools/android_test.sh
+(nntrainer) $ adb shell
+(adb) $ cd /data/local/tmp/nntr_android_test
+(adb) $ export LD_LIBRARY_PATH=.
+(adb) $ ./unittest_layers
+```
+
+- For more information, please refer to [tools](../tools/README.md)
+- [**Note**] Android unittest script builds NNTrainer to support GPU by default.

nntrainer/app_context.cpp

@@ -56,6 +56,7 @@
 #include <lr_scheduler_constant.h>
 #include <lr_scheduler_cosine.h>
 #include <lr_scheduler_exponential.h>
+#include <lr_scheduler_linear.h>
 #include <lr_scheduler_step.h>
 #include <lstm.h>
 #include <lstmcell.h>
@@ -252,6 +253,9 @@ static void add_default_object(AppContext &ac) {
                        CosineAnnealingLearningRateScheduler>,
                      CosineAnnealingLearningRateScheduler::type,
                      LRType::COSINE);
+  ac.registerFactory(
+    ml::train::createLearningRateScheduler<LinearLearningRateScheduler>,
+    LinearLearningRateScheduler::type, LRType::LINEAR);
 
   using LayerType = ml::train::LayerType;
   ac.registerFactory(nntrainer::createLayer<InputLayer>, InputLayer::type,

nntrainer/cl_context.cpp

@@ -23,6 +23,7 @@
 #include <reshape_cl.h>
 #include <rmsnorm_layer_cl.h>
 #include <swiglu_cl.h>
+#include <transpose_cl.h>
 
 namespace nntrainer {
 
@@ -51,6 +52,10 @@ static void add_default_object(ClContext &cc) {
 
   cc.registerFactory(nntrainer::createLayer<ConcatLayerCl>, ConcatLayerCl::type,
                      ml::train::LayerType::LAYER_CONCAT);
+
+  cc.registerFactory(nntrainer::createLayer<TransposeLayerCl>,
+                     TransposeLayerCl::type,
+                     ml::train::LayerType::LAYER_TRANSPOSE);
 }
 
 static void registerer(ClContext &cc) noexcept {
@@ -130,6 +135,7 @@ void ClContext::initBlasClKernels() {
   }
 
   registerClKernel(sgemv_cl_kernel_, "sgemv_cl");
+  registerClKernel(sgemv_cl_noTrans_kernel_, "sgemv_cl_noTrans");
   registerClKernel(dot_cl_kernel_, "dot_cl");
   registerClKernel(sgemm_cl_noTrans_kernel_, "sgemm_cl_noTrans");
   registerClKernel(sgemm_cl_transA_kernel_, "sgemm_cl_transA");
@@ -140,6 +146,7 @@ void ClContext::initBlasClKernels() {
 
 #ifdef ENABLE_FP16
   registerClKernel(sgemv_cl_kernel_fp16_, "sgemv_cl_fp16");
+  registerClKernel(sgemv_cl_noTrans_kernel_fp16_, "sgemv_cl_noTrans_fp16");
   registerClKernel(dot_cl_kernel_fp16_, "dot_cl_fp16");
   registerClKernel(sgemm_cl_noTrans_kernel_fp16_, "sgemm_cl_noTrans_fp16");
   registerClKernel(sgemm_cl_transA_kernel_fp16_, "sgemm_cl_transA_fp16");

nntrainer/layers/cl_layers/meson.build

@@ -5,6 +5,7 @@ cl_layer_sources = [
    'reshape_cl.cpp',
    'rmsnorm_layer_cl.cpp',
    'concat_cl.cpp',
+   'transpose_cl.cpp',
 ]
 
 foreach s : cl_layer_sources

nntrainer/layers/cl_layers/transpose_cl.cpp

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2024 Niket Agarwal <[email protected]>
+ *
+ * @file   transpose_cl.cpp
+ * @date   31 July 2024
+ * @brief  Implementation of transpose layer
+ * @see    https://github.com/nnstreamer/nntrainer
+ * @author Niket Agarwal <[email protected]>
+ * @bug    No known bugs except for NYI items
+ *
+ */
+
+#include "transpose_cl.h"
+#include <blas_kernel_interface.h>
+#include <iostream>
+#include <layer_context.h>
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <node_exporter.h>
+
+namespace nntrainer {
+
+static constexpr size_t SINGLE_INOUT_IDX = 0;
+
+void TransposeLayerCl::finalize(InitLayerContext &context) {
+  std::vector<TensorDim> dim = context.getInputDimensions();
+
+  for (unsigned int i = 0; i < dim.size(); ++i) {
+    if (dim[i].getDataLen() == 0) {
+      throw std::invalid_argument("Input dimension is not set");
+    }
+  }
+
+  context.setOutputDimensions(dim);
+}
+
+void TransposeLayerCl::forwarding(RunLayerContext &context, bool training) {
+  Tensor &in = context.getInput(SINGLE_INOUT_IDX);
+  Tensor &out = context.getOutput(SINGLE_INOUT_IDX);
+  // "1:0:2" is arbitrary
+  transposeCl("1:0:2", in, out);
+}
+
+void TransposeLayerCl::incremental_forwarding(RunLayerContext &context,
+                                              unsigned int from,
+                                              unsigned int to, bool training) {
+  Tensor &in = context.getInput(SINGLE_INOUT_IDX);
+  Tensor &out = context.getOutput(SINGLE_INOUT_IDX);
+  if (from) {
+    NNTR_THROW_IF(to - from != 1, std::invalid_argument)
+      << "incremental step size is not 1";
+    from = 0;
+    to = 1;
+  }
+  // "1:0:2" is arbitrary
+  transposeCl("1:0:2", in, out);
+}
+
+void TransposeLayerCl::calcDerivative(RunLayerContext &context) {
+  std::throw_with_nested(std::runtime_error("Training is not supported yet."));
+}
+
+void TransposeLayerCl::setProperty(const std::vector<std::string> &values) {
+  auto remain_props = loadProperties(values, transpose_props);
+  if (!remain_props.empty()) {
+    std::string msg = "[TransposeLayerCl] Unknown Layer Properties count " +
+                      std::to_string(values.size());
+    throw exception::not_supported(msg);
+  }
+}
+
+#ifdef PLUGGABLE
+
+Layer *create_transpose_layer_cl() {
+  auto layer = new TransposeLayerCl();
+  return layer;
+}
+
+void destroy_transpose_layer_cl(Layer *layer) { delete layer; }
+
+extern "C" {
+LayerPluggable ml_train_layer_pluggable{create_transpose_layer_cl,
+                                        destroy_transpose_layer_cl};
+}
+
+#endif
+
+} // namespace nntrainer

nntrainer/layers/cl_layers/transpose_cl.h

@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2024 Niket Agarwal <[email protected]>
+ *
+ * @file   transpose_cl.h
+ * @date   31 July 2024
+ * @brief  Implementation of transpose layer
+ * @see    https://github.com/nnstreamer/nntrainer
+ * @author Niket Agarwal <[email protected]>
+ * @bug    No known bugs except for NYI items
+ *
+ */
+
+#ifndef __TRANSPOSE_LAYER_CL_H__
+#define __TRANSPOSE_LAYER_CL_H__
+
+#include <common_properties.h>
+#include <layer_devel.h>
+#include <opencl_buffer.h>
+#include <opencl_kernel.h>
+
+namespace nntrainer {
+
+/**
+ * @brief A tranpose layer.
+ *
+ */
+class TransposeLayerCl final : public Layer {
+public:
+  /**
+   * @brief Construct a new transpose layer object
+   *
+   */
+  TransposeLayerCl() : Layer(), transpose_props(props::Print()) {}
+
+  /**
+   * @brief Destroy the transpose layer object
+   *
+   */
+  ~TransposeLayerCl() {}
+
+  /**
+   * @copydoc Layer::finalize(InitLayerContext &context)
+   */
+  void finalize(InitLayerContext &context) override;
+
+  /**
+   * @copydoc Layer::forwarding(RunLayerContext &context, bool training)
+   */
+  void forwarding(RunLayerContext &context, bool training) override;
+
+  /**
+   * @copydoc Layer::incremental_forwarding(RunLayerContext &context, unsigned
+   * int from, unsigned int to, bool training)
+   */
+  void incremental_forwarding(RunLayerContext &context, unsigned int from,
+                              unsigned int to, bool training) override;
+
+  /**
+   * @copydoc Layer::calcDerivative(RunLayerContext &context)
+   */
+  void calcDerivative(RunLayerContext &context) override;
+
+  /**
+   * @copydoc bool supportBackwarding() const
+   */
+  bool supportBackwarding() const override { return false; };
+
+  /**
+   * @copydoc Layer::exportTo(Exporter &exporter, ExportMethods method)
+   */
+  void exportTo(Exporter &exporter,
+                const ml::train::ExportMethods &method) const override{};
+
+  /**
+   * @copydoc Layer::getType()
+   */
+  const std::string getType() const override { return TransposeLayerCl::type; };
+
+  /**
+   * @copydoc Layer::setProperty(const std::vector<std::string> &values)
+   */
+  void setProperty(const std::vector<std::string> &values) override;
+
+  inline static const std::string type = "transpose";
+
+  static opencl::Kernel kernel_transpose_axis0;
+  static opencl::Kernel kernel_transpose_fp16_axis0;
+  static opencl::Kernel kernel_transpose_axis1;
+  static opencl::Kernel kernel_transpose_fp16_axis1;
+  static opencl::Kernel kernel_transpose_axis2;
+  static opencl::Kernel kernel_transpose_fp16_axis2;
+
+  std::tuple<props::Print> transpose_props; /**< transpose layer properties :
+                                            unit - number of output neurons */
+};
+} // namespace nntrainer
+
+#endif /* __TRANSPOSE_LAYER_CL_H__ */

nntrainer/layers/layer_devel.h

@@ -160,11 +160,6 @@ class Layer {
    */
   virtual void finalize(InitLayerContext &context) = 0;
 
-  /**
-   * @brief    Initialize the layer
-   */
-  virtual void initialize(RunLayerContext &context){};
-
   /**
    * @brief     Forward Propagation of a layer
    * @param     context Context of the layer

nntrainer/layers/layer_node.h

@@ -300,8 +300,6 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
    */
   InitLayerContext refinalize(const std::vector<TensorDim> &input_dims = {});
 
-  void initialize() override { layer->initialize(*run_context); }
-
   /**
    * @brief     Forward Propagation of a layer
    * @param     training true if training, false if inference

nntrainer/optimizers/lr_scheduler.h

@@ -30,7 +30,8 @@ enum LearningRateSchedulerType {
   CONSTANT = 0, /**< constant */
   EXPONENTIAL,  /**< exponentially decay */
   STEP,         /**< step wise decay */
-  COSINE        /**< cosine annealing */
+  COSINE,       /**< cosine annealing */
+  LINEAR        /**< linear decay */
 };
 
 /**
@@ -92,6 +93,11 @@ class LearningRateScheduler : public ml::train::LearningRateScheduler {
    * - min_learning_rate : float
    * - decay_steps : float
    *
+   * Linear Learning rate scheduler
+   * - max_learning_rate : float
+   * - min_learning_rate : float
+   * - decay_steps : positive integer
+   *
    * more to be added
    */