Merge pull request #1 from Russell91/cpp

Make apollo interface easier to use with C++

include/caffe/apollonet.hpp

@@ -27,8 +27,12 @@ class ApolloNet {
 
   Dtype ForwardLayer(const string& layer_param_string);
 
+  Dtype ForwardLayer(shared_ptr<Layer<Dtype> > layer);
+
   void BackwardLayer(const string& layer_name);
 
+  void Backward();
+
   void ResetForward() {
     active_layers_vec_.clear();
     active_layers_set_.clear();
@@ -48,6 +52,9 @@ class ApolloNet {
     CopyTrainedLayersFrom(param);
   }
 
+  void Update(Dtype lr, Dtype momentum, Dtype clip_gradients,
+    Dtype weight_decay);
+
   /// @brief returns the phase: TRAIN or TEST
   inline Phase phase() const { return phase_; }
 

include/caffe/data_layers.hpp

@@ -286,6 +286,53 @@ class MemoryDataLayer : public BaseDataLayer<Dtype> {
   bool has_new_data_;
 };
 
+/**
+ * @brief Provides data to the network from vector<cv::Mat> 
+ *
+ * TODO(dox): thorough documentation for Forward and proto params.
+ */
+template <typename Dtype>
+class MatDataLayer : public BaseDataLayer<Dtype> {
+ public:
+  explicit MatDataLayer(const LayerParameter& param)
+      : BaseDataLayer<Dtype>(param), data_(NULL) { }
+  virtual void DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top);
+
+  virtual inline const char* type() const { return "MatData"; }
+  virtual inline int ExactNumBottomBlobs() const { return 0; }
+  virtual inline int ExactNumTopBlobs() const { return 1; }
+
+  virtual void AddMatVector(const vector<cv::Mat>& mat_vector);
+
+  // Reset should accept const pointers, but can't, because the memory
+  //  will be given to Blob, which is mutable
+  void Reset(Dtype* data);
+  void set_batch_size(int new_size);
+
+  int batch_size() { return batch_size_; }
+  int channels() { return channels_; }
+  int height() { return height_; }
+  int width() { return width_; }
+
+ protected:
+  virtual void Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top);
+
+  shared_ptr<DataTransformer<Dtype> > data_transformer() {
+    if (!this->data_transformer_) {
+        this->data_transformer_.reset(
+            new DataTransformer<Dtype>(this->transform_param_, this->phase_));
+    }
+    return this->data_transformer_;
+  }
+
+  int batch_size_, channels_, height_, width_, size_;
+  Dtype* data_;
+  Blob<Dtype> added_data_;
+};
+
+
 template <typename Dtype>
 class NumpyDataLayer : public Layer<Dtype> {
  public:

include/caffe/layer_factory.hpp

@@ -39,6 +39,8 @@
 #ifndef CAFFE_LAYER_FACTORY_H_
 #define CAFFE_LAYER_FACTORY_H_
 
+#include <google/protobuf/text_format.h>
+
 #include <map>
 #include <string>
 
@@ -79,6 +81,14 @@ class LayerRegistry {
     return registry[type](param);
   }
 
+  // Get a layer using a prototxt which will be parsed into LayerParameter
+  static shared_ptr<Layer<Dtype> > CreateLayer(const string& prototxt) {
+    LayerParameter p;
+    bool success = google::protobuf::TextFormat::ParseFromString(prototxt, &p);
+    ASSERT(success, "Invalid prototxt string");
+    return CreateLayer(p);
+  }
+
  private:
   // Layer registry should never be instantiated - everything is done with its
   // static variables.

python/apollocaffe/proto/__init__.py

@@ -0,0 +1 @@
+import caffe_pb2

scripts/cpp_lint.py

@@ -1610,6 +1610,7 @@ def CheckCaffeDataLayerSetUp(filename, clean_lines, linenum, error):
   if ix >= 0 and (
        line.find('void DataLayer<Dtype>::LayerSetUp') != -1 or
        line.find('void ImageDataLayer<Dtype>::LayerSetUp') != -1 or
+       line.find('void MatDataLayer<Dtype>::LayerSetUp') != -1 or
        line.find('void MemoryDataLayer<Dtype>::LayerSetUp') != -1 or
        line.find('void WindowDataLayer<Dtype>::LayerSetUp') != -1):
       error(filename, linenum, 'caffe/data_layer_setup', 2,
@@ -1622,6 +1623,7 @@ def CheckCaffeDataLayerSetUp(filename, clean_lines, linenum, error):
        line.find('void Base') == -1 and
        line.find('void DataLayer<Dtype>::DataLayerSetUp') == -1 and
        line.find('void ImageDataLayer<Dtype>::DataLayerSetUp') == -1 and
+       line.find('void MatDataLayer<Dtype>::DataLayerSetUp') == -1 and
        line.find('void MemoryDataLayer<Dtype>::DataLayerSetUp') == -1 and
        line.find('void WindowDataLayer<Dtype>::DataLayerSetUp') == -1):
       error(filename, linenum, 'caffe/data_layer_setup', 2,

src/caffe/apollonet.cpp

@@ -17,6 +17,119 @@ template <typename Dtype>
 ApolloNet<Dtype>::ApolloNet() {
   Init();
 }
+template <typename Dtype>
+Dtype ApolloNet<Dtype>::ForwardLayer(shared_ptr<Layer<Dtype> > layer) {
+  /* This function will
+   * 1) Add the layer to the cache if it's new
+   * 2) Set up the top blobs
+   * 3) Set up the bottom blobs
+   * 4) Set up the parameters
+   * 5) Call the Forward function */
+
+  const LayerParameter& active_layer_param = layer->layer_param();
+  string layer_name = active_layer_param.name();
+
+  const bool new_layer = layers_map_.find(layer_name) == layers_map_.end();
+  if (new_layer) {
+    LOG(INFO) << "Adding Layer " << layer_name;
+    LOG(INFO) << active_layer_param.DebugString();
+    layers_map_[layer_name] = layer;
+    active_layers_set_.insert(layer_name);
+  }
+
+  active_layers_vec_.push_back(layer_name);
+  vector<Blob<Dtype>*> bottom_vec;
+  vector<Blob<Dtype>*> top_vec;
+
+  const vector<string>& bottom_names = bottom_blob_names_[layer_name];
+  bool reset_bottoms = (active_layer_param.bottom_size()
+      != bottom_names.size());
+  for (int i = 0; i < active_layer_param.bottom_size(); ++i) {
+    const string& blob_name = active_layer_param.bottom(i);
+    ASSERT(blobs_.find(blob_name) != blobs_.end(),
+      "Could not find bottom: '" << blob_name <<
+      "' for layer: " << layer_name);
+    if (bottom_names.size() > i &&
+        bottom_names[i] != blob_name) {
+      reset_bottoms = true;
+    }
+  }
+
+  if (new_layer || reset_bottoms) {
+    // Reset the bottom blobs
+    bottom_blobs_[layer_name].clear();
+    bottom_blob_names_[layer_name].clear();
+    for (int i = 0; i < active_layer_param.bottom_size(); ++i) {
+      const string& blob_name = active_layer_param.bottom(i);
+      shared_ptr<Blob<Dtype> > top_blob = blobs_[blob_name];
+      bottom_blob_names_[layer_name].push_back(blob_name);
+      shared_ptr<Blob<Dtype> > bottom_blob(
+        new Blob<Dtype>(top_blob->shape()));
+      bottom_blobs_[layer_name].push_back(bottom_blob);
+    }
+    layer->reset_bottoms(bottom_blob_names_[layer_name]);
+  }
+
+  for (int i = 0; i < active_layer_param.bottom_size(); ++i) {
+    // Reshape bottom_blobs to match their respective top blobs
+    const string& blob_name = active_layer_param.bottom(i);
+    shared_ptr<Blob<Dtype> > top_blob = blobs_[blob_name];
+    shared_ptr<Blob<Dtype> > bottom_blob = bottom_blobs_[layer_name][i];
+
+    bottom_blob->ReshapeLike(*top_blob);
+    bottom_blob->ShareData(*top_blob);
+    if (layer->in_place_layer() || !layer->overwrites_bottom_diffs()) {
+      // save memory when layer accumulates delta rather than overwriting
+      bottom_blob->ShareDiff(*top_blob);
+    }
+  }
+
+  for (int i = 0; i < active_layer_param.bottom_size(); ++i) {
+    bottom_vec.push_back(bottom_blobs_[layer_name][i].get());
+  }
+
+  ASSERT(layer->layer_param().top_size()
+      == active_layer_param.top_size(), "top vec cannot change");
+  for (int top_id = 0; top_id < active_layer_param.top_size(); ++top_id) {
+    ASSERT(layer->layer_param().top(top_id)
+      == active_layer_param.top(top_id), "top vec cannot change");
+  }
+
+  for (int top_id = 0; top_id < active_layer_param.top_size(); ++top_id) {
+    const string& blob_name = active_layer_param.top(top_id);
+    if (blobs_.find(blob_name) == blobs_.end()) {
+      shared_ptr<Blob<Dtype> > blob_pointer(new Blob<Dtype>());
+      blobs_[blob_name] = blob_pointer;
+    }
+    Blob<Dtype>* top_blob = blobs_[blob_name].get();
+    if (!layer->in_place_layer()) {
+      std::pair<set<string>::iterator, bool> ret;
+      ret = active_blobs_set_.insert(blob_name);
+      ASSERT(ret.second, "Top with name '"
+          << blob_name << "' is already used");
+    }
+    top_vec.push_back(top_blob);
+    if (top_blob->DiffInitialized() && !layer->is_loss()) {
+      // Zero out top_diffs, except for loss blobs, which never change
+      top_blob->SetDiffValues(0.);
+    }
+  }
+
+  if (new_layer) {
+    layer->SetUp(bottom_vec, top_vec);
+    AddLayerParams(layer);
+  }
+
+  for (int param_id = 0; param_id < layer->param_names().size(); ++param_id) {
+    const string& param_name = layer->param_names()[param_id];
+    active_params_set_.insert(param_name);
+  }
+
+  Dtype loss = 0;
+  layer->set_phase(phase_);
+  loss = layer->Forward(bottom_vec, top_vec);
+  return loss;
+}
 
 template <typename Dtype>
 Dtype ApolloNet<Dtype>::ForwardLayer(const string& layer_param_string) {
@@ -205,6 +318,38 @@ void ApolloNet<Dtype>::AddLayerParams(shared_ptr<Layer<Dtype> > layer) {
   }
 }
 
+template <typename Dtype>
+void ApolloNet<Dtype>::Backward() {
+  for (int i = active_layers_vec_.size() - 1; i >= 0; --i) {
+    BackwardLayer(active_layers_vec_[i]);
+  }
+}
+
+template <typename Dtype>
+void ApolloNet<Dtype>::Update(Dtype lr, Dtype momentum,
+  Dtype clip_gradients, Dtype weight_decay) {
+  Dtype diffnorm = DiffL2Norm();
+  Dtype clip_scale = 1;
+  if (clip_gradients > 0) {
+    if (diffnorm > clip_gradients) {
+      clip_scale = clip_gradients / diffnorm;
+    }
+  }
+  // iterate over active params
+  for (set<string>::iterator it = active_params_set_.begin();
+      it != active_params_set_.end(); ++it) {
+    const string& param_name = *it;
+    shared_ptr<Blob<Dtype> > cur_param = params_[param_name];
+    Dtype lr_new = lr * clip_scale * param_lr_mults()[param_name];
+    Dtype decay_new = weight_decay * param_decay_mults()[param_name];
+    const shared_ptr<Tensor<Dtype> > &cur_data = cur_param->data();
+    const shared_ptr<Tensor<Dtype> > &cur_diff = cur_param->diff();
+    cur_diff->AddMulFrom(*cur_data, decay_new);
+    cur_data->AddMulFrom(*cur_diff, -lr_new);
+    cur_param->scale_diff(momentum);
+  }
+}
+
 template <typename Dtype>
 void ApolloNet<Dtype>::BackwardLayer(const string& layer_name) {
   shared_ptr<Layer<Dtype> > layer = layers_map_[layer_name];

src/caffe/layers/mat_data_layer.cpp

@@ -0,0 +1,52 @@
+#include <opencv2/core/core.hpp>
+
+#include <vector>
+
+#include "caffe/data_layers.hpp"
+#include "caffe/layer.hpp"
+#include "caffe/util/io.hpp"
+
+namespace caffe {
+
+template <typename Dtype>
+void MatDataLayer<Dtype>::DataLayerSetUp(const vector<Blob<Dtype>*>& bottom,
+     const vector<Blob<Dtype>*>& top) { }
+
+template <typename Dtype>
+void MatDataLayer<Dtype>::AddMatVector(const vector<cv::Mat>& mat_vector) {
+  // reshape layers
+  batch_size_ = mat_vector.size();
+  channels_ = mat_vector[0].channels();
+  height_ = mat_vector[0].rows;
+  width_ = mat_vector[0].cols;
+  size_ = channels_ * height_ * width_;
+  CHECK_GT(batch_size_ * size_, 0) <<
+      "batch_size, channels, height, and width must be positive";
+  added_data_.Reshape(batch_size_, channels_, height_, width_);
+  // TODO: is this necessary
+  added_data_.cpu_data();
+
+  // Apply data transformations (mirror, scale, crop...)
+  data_transformer()->Transform(mat_vector, &added_data_);
+  Dtype* top_data = added_data_.mutable_cpu_data();
+  Reset(top_data);
+}
+
+template <typename Dtype>
+void MatDataLayer<Dtype>::Reset(Dtype* data) {
+  CHECK(data);
+  data_ = data;
+}
+
+template <typename Dtype>
+void MatDataLayer<Dtype>::Forward_cpu(const vector<Blob<Dtype>*>& bottom,
+      const vector<Blob<Dtype>*>& top) {
+  CHECK(data_) << "MatDataLayer needs to be initalized by calling Reset";
+  top[0]->Reshape(batch_size_, channels_, height_, width_);
+  top[0]->set_cpu_data(data_);
+}
+
+INSTANTIATE_CLASS(MatDataLayer);
+REGISTER_LAYER_CLASS(MatData);
+
+}  // namespace caffe