[layer] add pow operation layer

added a pow operation layer. there was an example of a pow layer in the custom layers, so I modified the key value of the custom pow layer to "custom_pow" in order to avoid duplication of layer key value. **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: Seungbaek Hong <[email protected]>
nnstreamer · Nov 20, 2024 · 03b88a4 · 03b88a4
1 parent 765bc11
commit 03b88a4
Show file tree

Hide file tree

Showing 19 changed files with 298 additions and 17 deletions.
diff --git a/Applications/Custom/LayerClient/jni/main.cpp b/Applications/Custom/LayerClient/jni/main.cpp
@@ -151,7 +151,7 @@ int api_model_run() {
     /// creating array of layers same as in `custom_layer_client.ini`
     layers = std::vector<std::shared_ptr<ml::train::Layer>>{
       ml::train::layer::Input({"name=inputlayer", "input_shape=1:1:100"}),
-      ml::train::createLayer("pow", {"name=powlayer", "exponent=3"}),
+      ml::train::createLayer("custom_pow", {"name=powlayer", "exponent=3"}),
       ml::train::layer::FullyConnected(
         {"name=outputlayer", "input_layers=powlayer", "unit=10",
          "bias_initializer=zeros", "activation=softmax"}),

diff --git a/Applications/Custom/LayerClient/res/custom_layer_client.ini b/Applications/Custom/LayerClient/res/custom_layer_client.ini
@@ -17,7 +17,7 @@ Input_Shape = 1:1:100
 
 [powlayer]
 input_layers = inputlayer
-Type = pow # AppContext sees PowLayer::getType() and use this to parse type
+Type = custom_pow # AppContext sees PowLayer::getType() and use this to parse type
 exponent = 3 # registering a custom property is done at int PowLayer::setProperty
 
 [outputlayer]

diff --git a/Applications/Custom/LayerPlugin/layer_plugin_pow_test.cpp b/Applications/Custom/LayerPlugin/layer_plugin_pow_test.cpp
@@ -21,7 +21,7 @@
 
 GTEST_PARAMETER_TEST(PowLayer, LayerPluginCommonTest,
                      ::testing::Values(std::make_tuple("libpow_layer.so",
-                                                       "pow")));
+                                                       "custom_pow")));
 
 auto semantic_pow =
   LayerSemanticsParamType(nntrainer::createLayer<custom::PowLayer>,

diff --git a/Applications/Custom/LayerPlugin/layer_plugin_test.cpp b/Applications/Custom/LayerPlugin/layer_plugin_test.cpp
@@ -29,9 +29,9 @@ TEST(AppContext, DlRegisterOpen_p) {
 
   ac.registerLayer("libpow_layer.so", NNTRAINER_PATH);
 
-  auto layer = ac.createObject<nntrainer::Layer>("pow");
+  auto layer = ac.createObject<nntrainer::Layer>("custom_pow");
 
-  EXPECT_EQ(layer->getType(), "pow");
+  EXPECT_EQ(layer->getType(), "custom_pow");
 }
 
 TEST(AppContext, DlRegisterWrongPath_n) {
@@ -49,9 +49,9 @@ TEST(AppContext, DlRegisterDirectory_p) {
 
   ac.registerPluggableFromDirectory(NNTRAINER_PATH);
 
-  auto layer = ac.createObject<nntrainer::Layer>("pow");
+  auto layer = ac.createObject<nntrainer::Layer>("custom_pow");
 
-  EXPECT_EQ(layer->getType(), "pow");
+  EXPECT_EQ(layer->getType(), "custom_pow");
 }
 
 TEST(AppContext, DlRegisterDirectory_n) {
@@ -64,8 +64,8 @@ TEST(AppContext, DlRegisterDirectory_n) {
 TEST(AppContext, DefaultEnvironmentPath_p) {
   /// as NNTRAINER_PATH is fed to the test, this should success without an
   /// error
-  std::shared_ptr<ml::train::Layer> l = ml::train::createLayer("pow");
-  EXPECT_EQ(l->getType(), "pow");
+  std::shared_ptr<ml::train::Layer> l = ml::train::createLayer("custom_pow");
+  EXPECT_EQ(l->getType(), "custom_pow");
   std::shared_ptr<nntrainer::LayerNode> lnode =
     std::static_pointer_cast<nntrainer::LayerNode>(l);
 

diff --git a/Applications/Custom/pow.h b/Applications/Custom/pow.h
@@ -77,7 +77,7 @@ class PowLayer final : public nntrainer::Layer {
    */
   void setProperty(const std::vector<std::string> &values) override;
 
-  inline static const std::string type = "pow";
+  inline static const std::string type = "custom_pow";
 
 private:
   float exponent;

diff --git a/api/ccapi/include/layer.h b/api/ccapi/include/layer.h
@@ -41,6 +41,7 @@ enum LayerType {
   LAYER_SUBTRACT = ML_TRAIN_LAYER_TYPE_SUBTRACT, /**< Subtract Layer type */
   LAYER_MULTIPLY = ML_TRAIN_LAYER_TYPE_MULTIPLY, /**< Multiply Layer type */
   LAYER_DIVIDE = ML_TRAIN_LAYER_TYPE_DIVIDE,     /**< Divide Layer type */
+  LAYER_POW = ML_TRAIN_LAYER_TYPE_POW,           /**< Pow Layer type */
   LAYER_FC = ML_TRAIN_LAYER_TYPE_FC,         /**< Fully Connected Layer type */
   LAYER_SWIGLU = ML_TRAIN_LAYER_TYPE_SWIGLU, /**< Swiglu Layer type */
   LAYER_BN = ML_TRAIN_LAYER_TYPE_BN, /**< Batch Normalization Layer type */
@@ -337,6 +338,14 @@ DivideLayer(const std::vector<std::string> &properties = {}) {
   return createLayer(LayerType::LAYER_DIVIDE, properties);
 }
 
+/**
+ * @brief Helper function to create pow layer
+ */
+inline std::unique_ptr<Layer>
+PowLayer(const std::vector<std::string> &properties = {}) {
+  return createLayer(LayerType::LAYER_POW, properties);
+}
+
 /**
  * @brief Helper function to create fully connected layer
  */

diff --git a/api/nntrainer-api-common.h b/api/nntrainer-api-common.h
@@ -62,14 +62,15 @@ typedef enum {
     27, /**< Layer Normalization Layer type (Since 7.0) */
   ML_TRAIN_LAYER_TYPE_POSITIONAL_ENCODING =
     28, /**< Positional Encoding Layer type (Since 7.0) */
-  ML_TRAIN_LAYER_TYPE_IDENTITY = 29, /**< Identity Layer type (Since 8.0) */
-  ML_TRAIN_LAYER_TYPE_SWIGLU = 30,   /**< Swiglu Layer type */
-  ML_TRAIN_LAYER_TYPE_WEIGHT = 31,   /**< Weight Layer type (Since 9.0)*/
-  ML_TRAIN_LAYER_TYPE_ADD = 32,      /**< Add Layer type (Since 9.0)*/
-  ML_TRAIN_LAYER_TYPE_SUBTRACT = 33, /**< Subtract Layer type (Since 9.0)*/
-  ML_TRAIN_LAYER_TYPE_MULTIPLY = 34, /**< Multiply Layer type (Since 9.0)*/
-  ML_TRAIN_LAYER_TYPE_DIVIDE = 35,   /**< Divide Layer type (Since 9.0)*/
+  ML_TRAIN_LAYER_TYPE_IDENTITY = 29,  /**< Identity Layer type (Since 8.0) */
+  ML_TRAIN_LAYER_TYPE_SWIGLU = 30,    /**< Swiglu Layer type */
+  ML_TRAIN_LAYER_TYPE_WEIGHT = 31,    /**< Weight Layer type (Since 9.0)*/
+  ML_TRAIN_LAYER_TYPE_ADD = 32,       /**< Add Layer type (Since 9.0)*/
+  ML_TRAIN_LAYER_TYPE_SUBTRACT = 33,  /**< Subtract Layer type (Since 9.0)*/
+  ML_TRAIN_LAYER_TYPE_MULTIPLY = 34,  /**< Multiply Layer type (Since 9.0)*/
+  ML_TRAIN_LAYER_TYPE_DIVIDE = 35,    /**< Divide Layer type (Since 9.0)*/
   ML_TRAIN_LAYER_TYPE_TRANSPOSE = 36, /**< Transpose Layer type */
+  ML_TRAIN_LAYER_TYPE_POW = 37,       /**< Pow Layer type (Since 9.0)*/
   ML_TRAIN_LAYER_TYPE_PREPROCESS_FLIP =
     300, /**< Preprocess flip Layer (Since 6.5) */
   ML_TRAIN_LAYER_TYPE_PREPROCESS_TRANSLATE =

diff --git a/nntrainer/app_context.cpp b/nntrainer/app_context.cpp
@@ -70,6 +70,7 @@
 #include <plugged_optimizer.h>
 #include <pooling2d_layer.h>
 #include <positional_encoding_layer.h>
+#include <pow_layer.h>
 #include <preprocess_flip_layer.h>
 #include <preprocess_l2norm_layer.h>
 #include <preprocess_translate_layer.h>
@@ -269,6 +270,8 @@ static void add_default_object(AppContext &ac) {
                      LayerType::LAYER_MULTIPLY);
   ac.registerFactory(nntrainer::createLayer<DivideLayer>, DivideLayer::type,
                      LayerType::LAYER_DIVIDE);
+  ac.registerFactory(nntrainer::createLayer<PowLayer>, PowLayer::type,
+                     LayerType::LAYER_POW);
   ac.registerFactory(nntrainer::createLayer<FullyConnectedLayer>,
                      FullyConnectedLayer::type, LayerType::LAYER_FC);
   ac.registerFactory(nntrainer::createLayer<BatchNormalizationLayer>,

diff --git a/nntrainer/layers/common_properties.cpp b/nntrainer/layers/common_properties.cpp
@@ -88,6 +88,8 @@ InputConnection::InputConnection(const Connection &value) :
 
 Epsilon::Epsilon(float value) { set(value); }
 
+Exponent::Exponent(float value) { set(value); }
+
 bool Epsilon::isValid(const float &value) const { return value > 0.0f; }
 
 Momentum::Momentum(float value) { set(value); }

diff --git a/nntrainer/layers/common_properties.h b/nntrainer/layers/common_properties.h
@@ -280,6 +280,22 @@ class Epsilon : public nntrainer::Property<float> {
   bool isValid(const float &value) const override;
 };
 
+/**
+ * @brief Exponent property, this is used for pow operation
+ *
+ */
+class Exponent : public nntrainer::Property<float> {
+
+public:
+  /**
+   * @brief Construct a new Exponent object
+   *
+   */
+  Exponent(float value = 1.0f);
+  static constexpr const char *key = "exponent"; /**< unique key to access */
+  using prop_tag = float_prop_tag;               /**< property type */
+};
+
 /**
  * @brief Momentum property, moving average in batch normalization layer
  *

diff --git a/nntrainer/layers/meson.build b/nntrainer/layers/meson.build
@@ -9,6 +9,7 @@ layer_sources = [
   'subtract_layer.cpp',
   'multiply_layer.cpp',
   'divide_layer.cpp',
+  'pow_layer.cpp',
   'addition_layer.cpp',
   'attention_layer.cpp',
   'mol_attention_layer.cpp',

diff --git a/nntrainer/layers/pow_layer.cpp b/nntrainer/layers/pow_layer.cpp
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2024 SeungBaek Hong <[email protected]>
+ *
+ * @file   pow_layer.cpp
+ * @date   20 Nov 2024
+ * @see    https://github.com/nnstreamer/nntrainer
+ * @author SeungBaek Hong <[email protected]>
+ * @bug    No known bugs except for NYI items
+ * @brief  This is pow layer class (operation layer)
+ *
+ */
+
+#include "common_properties.h"
+#include <nntrainer_error.h>
+#include <nntrainer_log.h>
+#include <node_exporter.h>
+#include <pow_layer.h>
+#include <util_func.h>
+
+#include <layer_context.h>
+
+namespace nntrainer {
+
+void PowLayer::finalize(InitLayerContext &context) {
+  context.setOutputDimensions({context.getInputDimensions()[0]});
+}
+
+void PowLayer::forwarding_operation(const Tensor &input, Tensor &hidden) {
+  float exp = std::get<props::Exponent>(pow_props).get();
+  input.pow(exp, hidden);
+}
+
+void PowLayer::calcDerivative(RunLayerContext &context) {
+  float exp = std::get<props::Exponent>(pow_props).get();
+  context.getOutgoingDerivative(0).copy(
+    context.getIncomingDerivative(SINGLE_INOUT_IDX)
+      .multiply(exp)
+      .multiply(context.getInput(0).pow(exp - 1.0f)));
+}
+
+void PowLayer::setProperty(const std::vector<std::string> &values) {
+  auto remain_props = loadProperties(values, pow_props);
+  if (!remain_props.empty()) {
+    std::string msg = "[PowLayer] Unknown Layer Properties count " +
+                      std::to_string(values.size());
+    throw exception::not_supported(msg);
+  }
+}
+} /* namespace nntrainer */
diff --git a/nntrainer/layers/pow_layer.h b/nntrainer/layers/pow_layer.h
@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: Apache-2.0
+/**
+ * Copyright (C) 2024 SeungBaek Hong <[email protected]>
+ *
+ * @file   pow_layer.h
+ * @date   20 Nov 2024
+ * @see    https://github.com/nnstreamer/nntrainer
+ * @author SeungBaek Hong <[email protected]>
+ * @bug    No known bugs except for NYI items
+ * @brief  This is pow layer class (operation layer)
+ *
+ */
+
+#ifndef __POW_LAYER_H__
+#define __POW_LAYER_H__
+#ifdef __cplusplus
+
+#include <common_properties.h>
+#include <layer_devel.h>
+#include <operation_layer.h>
+
+namespace nntrainer {
+
+/**
+ * @class Pow Layer
+ * @brief Pow Layer
+ */
+class PowLayer : public UnaryOperationLayer {
+public:
+  /**
+   * @brief Constructor of Pow Layer
+   */
+  PowLayer() :
+    UnaryOperationLayer(),
+    pow_props(props::Print(), props::InPlaceProp(), props::Exponent()),
+    support_backwarding(true) {}
+
+  /**
+   * @brief Destructor of Pow Layer
+   */
+  ~PowLayer(){};
+
+  /**
+   *  @brief  Move constructor of Pow Layer.
+   *  @param[in] PowLayer &&
+   */
+  PowLayer(PowLayer &&rhs) noexcept = default;
+
+  /**
+   * @brief  Move assignment operator.
+   * @parma[in] rhs PowLayer to be moved.
+   */
+  PowLayer &operator=(PowLayer &&rhs) = default;
+
+  /**
+   * @copydoc Layer::finalize(InitLayerContext &context)
+   */
+  void finalize(InitLayerContext &context) final;
+
+  /**
+   * @brief forwarding operation for pow
+   *
+   * @param input input tensor
+   * @param hidden tensor to store the result value
+   */
+  void forwarding_operation(const Tensor &input, Tensor &hidden) final;
+
+  /**
+   * @copydoc Layer::calcDerivative(RunLayerContext &context)
+   */
+  void calcDerivative(RunLayerContext &context) final;
+
+  /**
+   * @copydoc bool supportBackwarding() const
+   */
+  bool supportBackwarding() const final { return support_backwarding; };
+
+  /**
+   * @brief Initialize the in-place settings of the layer
+   * @return InPlaceType
+   */
+  InPlaceType initializeInPlace() final {
+    if (std::get<props::InPlaceProp>(pow_props).empty() ||
+        !std::get<props::InPlaceProp>(pow_props).get()) {
+      is_inplace = false;
+      support_backwarding = true;
+    } else {
+      is_inplace = true;
+      support_backwarding = false;
+    }
+
+    if (!supportInPlace())
+      return InPlaceType::NONE;
+    else
+      return InPlaceType::NON_RESTRICTING;
+  }
+
+  /**
+   * @copydoc Layer::exportTo(Exporter &exporter, ml::train::ExportMethods
+   * method)
+   */
+  void exportTo(Exporter &exporter,
+                const ml::train::ExportMethods &method) const final {}
+
+  /**
+   * @copydoc Layer::setProperty(const std::vector<std::string> &values)
+   */
+  void setProperty(const std::vector<std::string> &values) final;
+
+  /**
+   * @copydoc Layer::getType()
+   */
+  const std::string getType() const final { return PowLayer::type; };
+
+  std::tuple<props::Print, props::InPlaceProp, props::Exponent> pow_props;
+  bool support_backwarding; /**< support backwarding */
+
+  inline static const std::string type = "pow";
+};
+
+} // namespace nntrainer
+
+#endif /* __cplusplus */
+#endif /* __POW_LAYER_H__ */
diff --git a/nntrainer/utils/node_exporter.h b/nntrainer/utils/node_exporter.h
@@ -237,6 +237,7 @@ class Packed;
 class LossScaleForMixed;
 class InPlaceProp;
 class InPlaceDirectionProp;
+class Exponent;
 } // namespace props
 
 class LayerNode;

diff --git a/test/input_gen/genModelTests_v2.py b/test/input_gen/genModelTests_v2.py
@@ -518,6 +518,19 @@ def forward(self, inputs, labels):
         return out, loss
 
 
+class PowOperation(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.fc = torch.nn.Linear(2, 2)
+        self.loss = torch.nn.MSELoss()
+
+    def forward(self, inputs, labels):
+        out = self.fc(inputs[0])
+        out = out.pow(3)
+        loss = self.loss(out, labels[0])
+        return out, loss
+
+
 if __name__ == "__main__":
     record_v2(
         ReduceMeanLast(),
@@ -836,6 +849,16 @@ def forward(self, inputs, labels):
         name="multiply_operation",
     )
 
+    pow_operation = PowOperation()
+    record_v2(
+        pow_operation,
+        iteration=2,
+        input_dims=[(1, 2)],
+        input_dtype=[float],
+        label_dims=[(1, 2)],
+        name="pow_operation",
+    )
+
     # Function to check the created golden test file
     inspect_file("add_operation.nnmodelgolden")
     fc_mixed_training_nan_sgd = LinearMixedPrecisionNaNSGD()

diff --git a/test/input_gen/golden/unittest_models_v2.tar.gz b/test/input_gen/golden/unittest_models_v2.tar.gz