[ Property ] Add Tensor Type property in model

This PR enables the tensor type and tensor format in model property as "tensor_format=NHWC" or "tensor_type=FP16". This information goes to network_grap and layer node & manager. Then, each layer can get the model tensor type information and it can be used to request tensor or just using temporal tensor. Resolves: **Self evaluation:** 1. Build test: [X]Passed [ ]Failed [ ]Skipped 2. Run test: [X]Passed [ ]Failed [ ]Skipped Signed-off-by: jijoong.moon <[email protected]>
jijoongmoon · Jul 18, 2023 · ceeb3a9 · ceeb3a9
1 parent 4bea951
commit ceeb3a9
Show file tree

Hide file tree

Showing 38 changed files with 653 additions and 514 deletions.
diff --git a/api/ccapi/include/layer.h b/api/ccapi/include/layer.h
@@ -168,11 +168,11 @@ class Layer {
   virtual void setProperty(const std::vector<std::string> &values) = 0;
 
   /**
-   * @brief     Set Tensor Type : NCHW or NHWC
+   * @brief     Set Tensor format & data type
    * @note      This is used mainly for the unittest case which does not have
    * model.
    */
-  virtual void setTensorType(const std::string &values){};
+  virtual void setTensorType(std::array<const std::string, 2> &type){};
 
   /**
    * @brief     Get name of the layer

diff --git a/nntrainer/graph/network_graph.cpp b/nntrainer/graph/network_graph.cpp
@@ -81,17 +81,11 @@ int NetworkGraph::compile(const std::string &loss_type) {
 
   inPlaceOptimize();
 
-  TensorDim::Format type =
-    (getModelTensorType().compare("NCHW") ||
-     getModelTensorType().compare("nchw") || getModelTensorType().compare(""))
-      ? TensorDim::Format::NCHW
-      : TensorDim::Format::NHWC;
-
   for (auto iter = cbegin(); iter != cend(); iter++) {
     auto lnode = (*iter);
     /// @todo  later, we can set layer tensor type differenctly with model
     /// tensor type
-    lnode->setTensorType(getModelTensorType());
+    lnode->setTensorType(getTensorType());
   }
 
   status = checkCompiledGraph();

diff --git a/nntrainer/graph/network_graph.h b/nntrainer/graph/network_graph.h
@@ -49,7 +49,8 @@ class NetworkGraph {
     forward_iter_end(nullptr),
     optimize_memory(true),
     exec_mode(ExecutionMode::TRAIN),
-    model_tensor_type("NCHW") {}
+    tensor_format("NCHW"),
+    tensor_dtype("FP32") {}
 
   /**
    * @brief     Constructor of NeuralNetwork Graph Class
@@ -58,9 +59,10 @@ class NetworkGraph {
    */
   NetworkGraph(bool enable_swap, const std::string &swap_path = "",
                unsigned int lookahead = 0,
-               const std::string &tensor_type = "NCHW") :
+               const std::string &tensor_format_ = "NCHW",
+               const std::string &tensor_dtype_ = "FP32") :
     tensor_manager(std::make_shared<Manager>(enable_swap, swap_path, lookahead,
-                                             tensor_type)),
+                                             tensor_format_, tensor_dtype_)),
     graph(),
     compiled(false),
     batch_size(0),
@@ -69,7 +71,8 @@ class NetworkGraph {
     forward_iter_end(nullptr),
     optimize_memory(true),
     exec_mode(ExecutionMode::TRAIN),
-    model_tensor_type(tensor_type) {}
+    tensor_format(tensor_format_),
+    tensor_dtype(tensor_dtype_) {}
 
   /**
    * @brief   Destructor of the NeuralNetwork Graph class
@@ -373,7 +376,9 @@ class NetworkGraph {
    *
    * @return TensorDim::Format NCHW or NHWC
    */
-  std::string getModelTensorType() const { return model_tensor_type; };
+  std::array<const std::string, 2> getTensorType() {
+    return {std::move(tensor_format), std::move(tensor_dtype)};
+  };
 
   /**
    * @brief Flush data to the device
@@ -428,7 +433,9 @@ class NetworkGraph {
   ExecutionMode exec_mode; /**< execution mode with which the graph has been
                               currently set or previously set */
 
-  std::string model_tensor_type; /**< Model Tensor Type: NCHW or NHWC */
+  std::string tensor_format; /**< Model Tensor Format: NCHW or NHWC */
+
+  std::string tensor_dtype; /**< Model Tensor Type: FP32, FP16 */
 
   std::unordered_map<std::string, int>
     profile_keys; /**< profile keys based on the layer type */

diff --git a/nntrainer/layers/fc_layer.cpp b/nntrainer/layers/fc_layer.cpp
@@ -75,8 +75,15 @@ void FullyConnectedLayer::finalize(InitLayerContext &context) {
   /** set weight specifications */
   // @todo : This NCHW format setting is just temporal, it needs to be set by
   // global configuration
-  TensorDim bias_dim(1, 1, 1, unit, TensorDim::TensorType(getTensorType(), TensorDim::DataType::FP32) , 0b0001);
-  TensorDim weight_dim(1, 1, in_dim.width(), unit, TensorDim::TensorType(getTensorType(), TensorDim::DataType::FP32), 0b0011);
+  TensorDim bias_dim(
+    1, 1, 1, unit,
+    TensorDim::TensorType(getTensorFormat(), TensorDim::DataType::FP32),
+    0b0001);
+
+  TensorDim weight_dim(
+    1, 1, in_dim.width(), unit,
+    TensorDim::TensorType(getTensorFormat(), TensorDim::DataType::FP32),
+    0b0011);
 
   weight_idx[FCParams::weight] = context.requestWeight(
     weight_dim, weight_initializer, weight_regularizer,

diff --git a/nntrainer/layers/layer_devel.h b/nntrainer/layers/layer_devel.h
@@ -27,6 +27,7 @@
 #include <string>
 #include <vector>
 
+#include <base_properties.h>
 #include <common.h>
 #include <tensor_dim.h>
 
@@ -241,36 +242,64 @@ class Layer {
    */
   virtual bool supportBackwarding() const = 0;
 
+  /**
+   * @brief Set the Tensor format for the layer
+   * @param     Tensor format : TensorDim::Format::NCHW or
+   * TneosrDim::Format::NHWC
+   */
+  virtual void setTensorFormat(
+    ml::train::TensorDim::Format form = ml::train::TensorDim::Format::NCHW) {
+    tensor_format = form;
+  }
+
   /**
    * @brief Set the Tensor Type for the layer
-   * @param     Tensor Type : TensorDim::Format::NCHW or TneosrDim::Format::NHWC
+   * @param     Tensor Type : FP32, FP16
    */
-  virtual void setTensorType(
-    ml::train::TensorDim::Format type = ml::train::TensorDim::Format::NCHW) {
-    tensor_type = type;
+
+  virtual void setTensorDataType(
+    ml::train::TensorDim::DataType ty = ml::train::TensorDim::DataType::FP32) {
+    tensor_dtype = ty;
   }
 
   /**
    * @brief set the Tensor Type for the layer
    * @param     Tensor Type : NCHW or NHWC
    */
-  void setTensorType(const std::string &values) {
-    if (values.compare("NCHW") || values.compare("nchw")) {
-      tensor_type = ml::train::TensorDim::Format::NCHW;
+  void setTensorType(std::array<const std::string, 2> t_type) {
+    if (t_type[0].compare("NCHW") == 0 || t_type[0].compare("nchw") == 0) {
+      tensor_format = ml::train::TensorDim::Format::NCHW;
     } else {
-      tensor_type = ml::train::TensorDim::Format::NHWC;
+      tensor_format = ml::train::TensorDim::Format::NHWC;
     }
+
+    nntrainer::props::TensorDataType type_;
+
+    from_string(t_type[1], type_);
+
+    tensor_dtype = type_;
   }
 
   /**
-   * @brief get the Tensor Type for the layer
-   * @return     Tensor Type : TensorDim::Format::NCHW or
+   * @brief get the Tensor Format for the layer
+   * @return     Tensor Format : TensorDim::Format::NCHW or
    * TneosrDim::Format::NHWC
    */
-  virtual ml::train::TensorDim::Format getTensorType() { return tensor_type; }
+  virtual ml::train::TensorDim::Format getTensorFormat() {
+    return tensor_format;
+  }
+
+  /**
+   * @brief get the Tensor Type for the layer
+   * @return     Tensor Type : FP16, Fp32
+   */
+  virtual ml::train::TensorDim::DataType getTensorDataType() {
+    return tensor_dtype;
+  }
 
 private:
-  ml::train::TensorDim::Format tensor_type;
+  ml::train::TensorDim::Format tensor_format;
+  ml::train::TensorDim::DataType tensor_dtype;
 };
 
 /// @todo Decide where to put and how to implement(#986)

diff --git a/nntrainer/layers/layer_node.cpp b/nntrainer/layers/layer_node.cpp
@@ -247,11 +247,12 @@ void LayerNode::setOutputConnection(unsigned nth, const std::string &name,
   con = std::make_unique<Connection>(name, index);
 }
 
-void LayerNode::setTensorType(const std::string type_) {
-  TensorDim::Format type = (type_.compare("NCHW") || type_.compare("nchw"))
-                             ? TensorDim::Format::NCHW
-                             : TensorDim::Format::NHWC;
-  getLayer()->setTensorType(type);
+void LayerNode::setTensorType(const std::string form_, const std::string ty_) {
+  setTensorType({std::move(form_), std::move(ty_)});
+}
+
+void LayerNode::setTensorType(std::array<const std::string, 2> t_type) {
+  getLayer()->setTensorType(t_type);
 }
 
 const std::string LayerNode::getName() const noexcept {

diff --git a/nntrainer/layers/layer_node.h b/nntrainer/layers/layer_node.h
@@ -804,9 +804,18 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
   /**
    * @brief Set Tensor type for layer
    *
-   * @param type NCHW : NHWC
+   * @param format NCHW : NHWC
+   * @param type FP16, FP32
    */
-  void setTensorType(const std::string type_ = "NCHW");
+  void setTensorType(const std::string form_ = "NCHW",
+                     const std::string type_ = "FP32");
+  /**
+   * @brief Set Tensor type for layer
+   *
+   * @param format NCHW : NHWC
+   * @param type FP16, FP32
+   */
+  void setTensorType(std::array<const std::string, 2> t_type);
 
 private:
   /**
@@ -835,7 +844,9 @@ class LayerNode final : public ml::train::Layer, public GraphNode {
   std::vector<std::unique_ptr<Connection>>
     output_connections; /**< output layer names */
 
-  TensorDim::Format tensor_type;
+  TensorDim::Format tensor_format;
+
+  TensorDim::DataType tensor_dtype;
 
 #ifdef ENABLE_TEST
   /**

diff --git a/nntrainer/models/model_common_properties.cpp b/nntrainer/models/model_common_properties.cpp
@@ -36,7 +36,4 @@ MemorySwapPath::MemorySwapPath(const std::string &value) { set(value); }
 MemorySwapLookahead::MemorySwapLookahead(const unsigned int &value) {
   set(value);
 }
-
-ModelTensorType::ModelTensorType(const std::string &value) { set(value); }
-
 } // namespace nntrainer::props
diff --git a/nntrainer/models/model_common_properties.h b/nntrainer/models/model_common_properties.h
@@ -179,22 +179,6 @@ class MemorySwapLookahead : public Property<unsigned int> {
   MemorySwapLookahead(const unsigned int &value = 0);
 };
 
-/**
- * @brief model tensor type : NCHW or NHWC
- *
- */
-class ModelTensorType : public Property<std::string> {
-public:
-  static constexpr const char *key = "tensor_type"; /**< unique key to access */
-  using prop_tag = str_prop_tag;                    /**< property type */
-
-  /**
-   * @brief Constructor
-   *
-   * @param value value to set, defaults to false
-   */
-  ModelTensorType(const std::string &value = "NCHW");
-};
 
 } // namespace nntrainer::props
 

diff --git a/nntrainer/models/neuralnet.cpp b/nntrainer/models/neuralnet.cpp
@@ -66,11 +66,11 @@ namespace nntrainer {
 
 NeuralNetwork::NeuralNetwork() :
   model_props(props::LossType(), {}, {}, props::ClipGradByGlobalNorm()),
-  model_flex_props(props::Epochs(), props::TrainingBatchSize(),
-                   props::SavePath(), props::ContinueTrain(),
-                   props::SaveBestPath(), props::MemoryOptimization(),
-                   props::MemorySwap(), props::MemorySwapPath(),
-                   props::MemorySwapLookahead(), props::ModelTensorType()),
+  model_flex_props(
+    props::Epochs(), props::TrainingBatchSize(), props::SavePath(),
+    props::ContinueTrain(), props::SaveBestPath(), props::MemoryOptimization(),
+    props::MemorySwap(), props::MemorySwapPath(), props::MemorySwapLookahead(),
+    props::TensorFormat(), props::TensorDataType()),
   load_path(std::string()),
   epoch_idx(0),
   iter(0),
@@ -84,11 +84,11 @@ NeuralNetwork::NeuralNetwork() :
 
 NeuralNetwork::NeuralNetwork(AppContext app_context_) :
   model_props(props::LossType(), {}, {}, props::ClipGradByGlobalNorm()),
-  model_flex_props(props::Epochs(), props::TrainingBatchSize(),
-                   props::SavePath(), props::ContinueTrain(),
-                   props::SaveBestPath(), props::MemoryOptimization(),
-                   props::MemorySwap(), props::MemorySwapPath(),
-                   props::MemorySwapLookahead(), props::ModelTensorType()),
+  model_flex_props(
+    props::Epochs(), props::TrainingBatchSize(), props::SavePath(),
+    props::ContinueTrain(), props::SaveBestPath(), props::MemoryOptimization(),
+    props::MemorySwap(), props::MemorySwapPath(), props::MemorySwapLookahead(),
+    props::TensorFormat(), props::TensorDataType()),
   load_path(std::string()),
   epoch_idx(0),
   iter(0),
@@ -173,11 +173,14 @@ int NeuralNetwork::compile() {
   unsigned int lookahead =
     std::get<props::MemorySwapLookahead>(model_flex_props);
 
+  const std::string tensor_format =
+    std::get<props::TensorFormat>(model_flex_props);
+
   const std::string tensor_type =
-    std::get<props::ModelTensorType>(model_flex_props);
+    to_string(std::get<props::TensorDataType>(model_flex_props));
 
-  model_graph =
-    NetworkGraph(memory_swap, memory_swap_path, lookahead, tensor_type);
+  model_graph = NetworkGraph(memory_swap, memory_swap_path, lookahead,
+                             tensor_format, tensor_type);
 
   model_graph.setMemoryOptimizations(
     std::get<props::MemoryOptimization>(model_flex_props));

diff --git a/nntrainer/models/neuralnet.h b/nntrainer/models/neuralnet.h
@@ -557,7 +557,7 @@ class NeuralNetwork : public ml::train::Model {
                props::ContinueTrain, props::SaveBestPath,
                props::MemoryOptimization, props::MemorySwap,
                props::MemorySwapPath, props::MemorySwapLookahead,
-               props::ModelTensorType>;
+               props::TensorFormat, props::TensorDataType>;
   using RigidPropTypes =
     std::tuple<props::LossType, std::vector<props::InputConnection>,
                std::vector<props::LabelLayer>, props::ClipGradByGlobalNorm>;

diff --git a/nntrainer/tensor/manager.h b/nntrainer/tensor/manager.h
@@ -133,18 +133,21 @@ class Manager {
   Manager() :
     enable_optimizations(true),
     swap_lookahead(0),
-    tensor_type("nchw") {}
+    tensor_format("nchw"),
+    tensor_dtype("fp32") {}
 
   /**
    * @brief     Constructor of Manager
    */
   Manager(bool enable_swap, const std::string &swap_path = "",
-          unsigned int lookahead = 0, const std::string tensor_type_ = "nchw") :
+          unsigned int lookahead = 0, const std::string tensor_format_ = "nchw",
+          const std::string tensor_dtype_ = "fp32") :
     weight_pool(enable_swap, swap_path, "weight_pool"),
     tensor_pool(enable_swap, swap_path, "tensor_pool"),
     enable_optimizations(true),
     swap_lookahead(lookahead),
-    tensor_type(tensor_type_) {}
+    tensor_format(tensor_format_),
+    tensor_dtype(tensor_dtype_) {}
 
   /**
    * @brief Construct a new Manager object (deleted)
@@ -505,7 +508,9 @@ class Manager {
 
   unsigned int swap_lookahead; /** lookahead for memory swap */
 
-  std::string tensor_type;
+  std::string tensor_format;
+
+  std::string tensor_dtype;
 
   /**
    * @brief Finalize the given tensor pool