From af057eb99fd088e6ced921bedb78316770b6b16f Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sat, 15 Aug 2015 18:13:40 -0400 Subject: [PATCH 1/9] added method to add layers directly --- include/caffe/apollonet.hpp | 2 + src/caffe/apollonet.cpp | 113 ++++++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/include/caffe/apollonet.hpp b/include/caffe/apollonet.hpp index 584d82b7b21..d1d1aee10fe 100644 --- a/include/caffe/apollonet.hpp +++ b/include/caffe/apollonet.hpp @@ -27,6 +27,8 @@ class ApolloNet { Dtype ForwardLayer(const string& layer_param_string); + Dtype ForwardLayer(shared_ptr > layer); + void BackwardLayer(const string& layer_name); void ResetForward() { diff --git a/src/caffe/apollonet.cpp b/src/caffe/apollonet.cpp index 9757683cfd4..12b3039c730 100644 --- a/src/caffe/apollonet.cpp +++ b/src/caffe/apollonet.cpp @@ -17,6 +17,119 @@ template ApolloNet::ApolloNet() { Init(); } +template +Dtype ApolloNet::ForwardLayer(shared_ptr > layer) { + /* This function will + * 1) Add the layer to the cache if it's new + * 2) Set up the top blobs + * 3) Set up the bottom blobs + * 4) Set up the parameters + * 5) Call the Forward function */ + + const LayerParameter& active_layer_param = layer->layer_param(); + string layer_name = active_layer_param.name(); + active_layers_vec_.push_back(layer_name); + vector*> bottom_vec; + vector*> top_vec; + + //const LayerParameter& active_layer_param = layer->layer_param(); + const bool new_layer = layers_map_.find(layer_name) == layers_map_.end(); + if (new_layer) { + layers_map_[layer_name] = layer; + active_layers_set_.insert(layer_name); + LOG(INFO) << "Adding Layer " << layer_name; + LOG(INFO) << active_layer_param.DebugString(); + } + + const vector& bottom_names = bottom_blob_names_[layer_name]; + bool reset_bottoms = (active_layer_param.bottom_size() + != bottom_names.size()); + for (int i = 0; i < active_layer_param.bottom_size(); ++i) { + const string& blob_name = active_layer_param.bottom(i); + ASSERT(blobs_.find(blob_name) != blobs_.end(), + "Could not find bottom: '" << blob_name << + "' for layer: " << layer_name); + if (bottom_names.size() > i && + bottom_names[i] != blob_name) { + reset_bottoms = true; + } + } + + if (new_layer || reset_bottoms) { + // Reset the bottom blobs + bottom_blobs_[layer_name].clear(); + bottom_blob_names_[layer_name].clear(); + for (int i = 0; i < active_layer_param.bottom_size(); ++i) { + const string& blob_name = active_layer_param.bottom(i); + shared_ptr > top_blob = blobs_[blob_name]; + bottom_blob_names_[layer_name].push_back(blob_name); + shared_ptr > bottom_blob( + new Blob(top_blob->shape())); + bottom_blobs_[layer_name].push_back(bottom_blob); + } + layer->reset_bottoms(bottom_blob_names_[layer_name]); + } + + for (int i = 0; i < active_layer_param.bottom_size(); ++i) { + // Reshape bottom_blobs to match their respective top blobs + const string& blob_name = active_layer_param.bottom(i); + shared_ptr > top_blob = blobs_[blob_name]; + shared_ptr > bottom_blob = bottom_blobs_[layer_name][i]; + + bottom_blob->ReshapeLike(*top_blob); + bottom_blob->ShareData(*top_blob); + if (layer->in_place_layer() || !layer->overwrites_bottom_diffs()) { + // save memory when layer accumulates delta rather than overwriting + bottom_blob->ShareDiff(*top_blob); + } + } + + for (int i = 0; i < active_layer_param.bottom_size(); ++i) { + bottom_vec.push_back(bottom_blobs_[layer_name][i].get()); + } + + ASSERT(layer->layer_param().top_size() + == active_layer_param.top_size(), "top vec cannot change"); + for (int top_id = 0; top_id < active_layer_param.top_size(); ++top_id) { + ASSERT(layer->layer_param().top(top_id) + == active_layer_param.top(top_id), "top vec cannot change"); + } + + for (int top_id = 0; top_id < active_layer_param.top_size(); ++top_id) { + const string& blob_name = active_layer_param.top(top_id); + if (blobs_.find(blob_name) == blobs_.end()) { + shared_ptr > blob_pointer(new Blob()); + blobs_[blob_name] = blob_pointer; + } + Blob* top_blob = blobs_[blob_name].get(); + if (!layer->in_place_layer()) { + std::pair::iterator, bool> ret; + ret = active_blobs_set_.insert(blob_name); + ASSERT(ret.second, "Top with name '" + << blob_name << "' is already used"); + } + top_vec.push_back(top_blob); + if (top_blob->DiffInitialized() && !layer->is_loss()) { + // Zero out top_diffs, except for loss blobs, which never change + top_blob->SetDiffValues(0.); + } + } + + if (new_layer) { + layer->SetUp(bottom_vec, top_vec); + AddLayerParams(layer); + } + + for (int param_id = 0; param_id < layer->param_names().size(); ++param_id) { + const string& param_name = layer->param_names()[param_id]; + active_params_set_.insert(param_name); + } + + Dtype loss = 0; + layer->set_phase(phase_); + loss = layer->Forward(bottom_vec, top_vec); + return loss; +} template Dtype ApolloNet::ForwardLayer(const string& layer_param_string) { From 04acc22b2d2e5d7c38242de47d0c60af03389611 Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sat, 15 Aug 2015 22:51:17 -0400 Subject: [PATCH 2/9] added layer construction from prototxt string --- include/caffe/layer_factory.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/caffe/layer_factory.hpp b/include/caffe/layer_factory.hpp index 2fcd93869a0..1b1764f21ca 100644 --- a/include/caffe/layer_factory.hpp +++ b/include/caffe/layer_factory.hpp @@ -39,6 +39,8 @@ #ifndef CAFFE_LAYER_FACTORY_H_ #define CAFFE_LAYER_FACTORY_H_ +#include + #include #include @@ -78,6 +80,14 @@ class LayerRegistry { << " (known types: " << LayerTypeList() << ")"; return registry[type](param); } + + // Get a layer using a prototxt which will be parsed into LayerParameter + static shared_ptr > CreateLayer(const string& prototxt) { + LayerParameter param; + bool success = google::protobuf::TextFormat::ParseFromString(prototxt, ¶m); + ASSERT(success, "Invalid prototxt string"); + return CreateLayer(param); + } private: // Layer registry should never be instantiated - everything is done with its From 33f3592b2d492f678aa8eae9571248206a0e672f Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sat, 15 Aug 2015 22:52:05 -0400 Subject: [PATCH 3/9] added mat data layer --- include/caffe/data_layers.hpp | 47 +++++++++++++++++++++++++ src/caffe/layers/mat_data_layer.cpp | 53 +++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 src/caffe/layers/mat_data_layer.cpp diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp index 9027cb7a0e6..3b552b3de7a 100644 --- a/include/caffe/data_layers.hpp +++ b/include/caffe/data_layers.hpp @@ -286,6 +286,53 @@ class MemoryDataLayer : public BaseDataLayer { bool has_new_data_; }; +/** + * @brief Provides data to the network from vector + * + * TODO(dox): thorough documentation for Forward and proto params. + */ +template +class MatDataLayer : public BaseDataLayer { + public: + explicit MatDataLayer(const LayerParameter& param) + : BaseDataLayer(param), data_(NULL) { } + virtual void DataLayerSetUp(const vector*>& bottom, + const vector*>& top); + + virtual inline const char* type() const { return "MatData"; } + virtual inline int ExactNumBottomBlobs() const { return 0; } + virtual inline int ExactNumTopBlobs() const { return 1; } + + virtual void AddMatVector(const vector& mat_vector); + + // Reset should accept const pointers, but can't, because the memory + // will be given to Blob, which is mutable + void Reset(Dtype* data); + void set_batch_size(int new_size); + + int batch_size() { return batch_size_; } + int channels() { return channels_; } + int height() { return height_; } + int width() { return width_; } + + protected: + virtual void Forward_cpu(const vector*>& bottom, + const vector*>& top); + + shared_ptr > data_transformer(){ + if (!this->data_transformer_){ + this->data_transformer_.reset( + new DataTransformer(this->transform_param_, this->phase_)); + } + return this->data_transformer_; + } + + int batch_size_, channels_, height_, width_, size_; + Dtype* data_; + Blob added_data_; +}; + + template class NumpyDataLayer : public Layer { public: diff --git a/src/caffe/layers/mat_data_layer.cpp b/src/caffe/layers/mat_data_layer.cpp new file mode 100644 index 00000000000..21463531618 --- /dev/null +++ b/src/caffe/layers/mat_data_layer.cpp @@ -0,0 +1,53 @@ +#include + +#include + +#include "caffe/data_layers.hpp" +#include "caffe/layer.hpp" +#include "caffe/util/io.hpp" + +namespace caffe { + +template +void MatDataLayer::DataLayerSetUp(const vector*>& bottom, + const vector*>& top) { } + +template +void MatDataLayer::AddMatVector(const vector& mat_vector) { + // reshape layers + batch_size_ = mat_vector.size(); + channels_ = mat_vector[0].channels(); + height_ = mat_vector[0].rows; + width_ = mat_vector[0].cols; + size_ = channels_ * height_ * width_; + CHECK_GT(batch_size_ * size_, 0) << + "batch_size, channels, height, and width must be specified and" + " positive in memory_data_param"; + added_data_.Reshape(batch_size_, channels_, height_, width_); + //TODO: is this necessary + added_data_.cpu_data(); + + // Apply data transformations (mirror, scale, crop...) + data_transformer()->Transform(mat_vector, &added_data_); + Dtype* top_data = added_data_.mutable_cpu_data(); + Reset(top_data); +} + +template +void MatDataLayer::Reset(Dtype* data) { + CHECK(data); + data_ = data; +} + +template +void MatDataLayer::Forward_cpu(const vector*>& bottom, + const vector*>& top) { + CHECK(data_) << "MatDataLayer needs to be initalized by calling Reset"; + top[0]->Reshape(batch_size_, channels_, height_, width_); + top[0]->set_cpu_data(data_); +} + +INSTANTIATE_CLASS(MatDataLayer); +REGISTER_LAYER_CLASS(MatData); + +} // namespace caffe From 1af74e0298c6d7f2d7134f599508c5474e4871a0 Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sat, 15 Aug 2015 23:03:57 -0400 Subject: [PATCH 4/9] possible bug fix --- python/apollocaffe/proto/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/apollocaffe/proto/__init__.py b/python/apollocaffe/proto/__init__.py index e69de29bb2d..adede953ea4 100644 --- a/python/apollocaffe/proto/__init__.py +++ b/python/apollocaffe/proto/__init__.py @@ -0,0 +1 @@ +import caffe_pb2 From 87915f42ff4780b6e45617df44810c13fab09b78 Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sat, 15 Aug 2015 23:45:52 -0400 Subject: [PATCH 5/9] added backward method to backprop in c++ --- include/caffe/apollonet.hpp | 2 ++ src/caffe/apollonet.cpp | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/include/caffe/apollonet.hpp b/include/caffe/apollonet.hpp index d1d1aee10fe..415ed58711e 100644 --- a/include/caffe/apollonet.hpp +++ b/include/caffe/apollonet.hpp @@ -31,6 +31,8 @@ class ApolloNet { void BackwardLayer(const string& layer_name); + void Backward(); + void ResetForward() { active_layers_vec_.clear(); active_layers_set_.clear(); diff --git a/src/caffe/apollonet.cpp b/src/caffe/apollonet.cpp index 12b3039c730..ece70c467ad 100644 --- a/src/caffe/apollonet.cpp +++ b/src/caffe/apollonet.cpp @@ -28,19 +28,20 @@ Dtype ApolloNet::ForwardLayer(shared_ptr > layer) { const LayerParameter& active_layer_param = layer->layer_param(); string layer_name = active_layer_param.name(); - active_layers_vec_.push_back(layer_name); - vector*> bottom_vec; - vector*> top_vec; //const LayerParameter& active_layer_param = layer->layer_param(); const bool new_layer = layers_map_.find(layer_name) == layers_map_.end(); if (new_layer) { - layers_map_[layer_name] = layer; - active_layers_set_.insert(layer_name); LOG(INFO) << "Adding Layer " << layer_name; LOG(INFO) << active_layer_param.DebugString(); + layers_map_[layer_name] = layer; + active_layers_set_.insert(layer_name); } + active_layers_vec_.push_back(layer_name); + vector*> bottom_vec; + vector*> top_vec; + const vector& bottom_names = bottom_blob_names_[layer_name]; bool reset_bottoms = (active_layer_param.bottom_size() != bottom_names.size()); @@ -318,6 +319,13 @@ void ApolloNet::AddLayerParams(shared_ptr > layer) { } } +template +void ApolloNet::Backward() { + for (int i = active_layers_vec_.size() - 1; i >= 0; --i){ + BackwardLayer(active_layers_vec_[i]); + } +} + template void ApolloNet::BackwardLayer(const string& layer_name) { shared_ptr > layer = layers_map_[layer_name]; From 1c44024f64338bf25be790ef1a5089f4fa873bf1 Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sun, 16 Aug 2015 01:18:24 -0400 Subject: [PATCH 6/9] added method to update parameters --- include/caffe/apollonet.hpp | 2 ++ src/caffe/apollonet.cpp | 23 +++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/include/caffe/apollonet.hpp b/include/caffe/apollonet.hpp index 415ed58711e..f7c166b46b4 100644 --- a/include/caffe/apollonet.hpp +++ b/include/caffe/apollonet.hpp @@ -52,6 +52,8 @@ class ApolloNet { CopyTrainedLayersFrom(param); } + void Update(Dtype lr, Dtype momentum, Dtype clip_gradients, Dtype weight_decay); + /// @brief returns the phase: TRAIN or TEST inline Phase phase() const { return phase_; } diff --git a/src/caffe/apollonet.cpp b/src/caffe/apollonet.cpp index ece70c467ad..17d70bf4b44 100644 --- a/src/caffe/apollonet.cpp +++ b/src/caffe/apollonet.cpp @@ -326,6 +326,29 @@ void ApolloNet::Backward() { } } +template +void ApolloNet::Update(Dtype lr, Dtype momentum, Dtype clip_gradients, Dtype weight_decay) { + Dtype diffnorm = DiffL2Norm(); + Dtype clip_scale = 1; + if (clip_gradients > 0){ + if (diffnorm > clip_gradients){ + clip_scale = clip_gradients / diffnorm; + } + } + + // iterate over active params + for (set::iterator it = active_params_set_.begin(); + it != active_params_set_.end(); ++it) { + const string& param_name = *it; + shared_ptr > curParam = params_[param_name]; + const shared_ptr > &curData = curParam->data(); + const shared_ptr > &curDiff = curParam->diff(); + curDiff->AddMulFrom(*curData, weight_decay); + curData->AddMulFrom(*curDiff, -lr); + curParam->scale_diff(momentum); + } +} + template void ApolloNet::BackwardLayer(const string& layer_name) { shared_ptr > layer = layers_map_[layer_name]; From 91e9bf9040a9bc81df35b326131f093ea3fca4f8 Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sun, 16 Aug 2015 09:46:15 -0400 Subject: [PATCH 7/9] fixed update() method --- src/caffe/apollonet.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/caffe/apollonet.cpp b/src/caffe/apollonet.cpp index 17d70bf4b44..f9fdf4df7c4 100644 --- a/src/caffe/apollonet.cpp +++ b/src/caffe/apollonet.cpp @@ -29,7 +29,6 @@ Dtype ApolloNet::ForwardLayer(shared_ptr > layer) { const LayerParameter& active_layer_param = layer->layer_param(); string layer_name = active_layer_param.name(); - //const LayerParameter& active_layer_param = layer->layer_param(); const bool new_layer = layers_map_.find(layer_name) == layers_map_.end(); if (new_layer) { LOG(INFO) << "Adding Layer " << layer_name; @@ -319,32 +318,33 @@ void ApolloNet::AddLayerParams(shared_ptr > layer) { } } -template +template void ApolloNet::Backward() { - for (int i = active_layers_vec_.size() - 1; i >= 0; --i){ + for (int i = active_layers_vec_.size() - 1; i >= 0; --i) { BackwardLayer(active_layers_vec_[i]); } } -template -void ApolloNet::Update(Dtype lr, Dtype momentum, Dtype clip_gradients, Dtype weight_decay) { +template +void ApolloNet::Update(Dtype lr, Dtype momentum, + Dtype clip_gradients, Dtype weight_decay) { Dtype diffnorm = DiffL2Norm(); Dtype clip_scale = 1; - if (clip_gradients > 0){ - if (diffnorm > clip_gradients){ + if (clip_gradients > 0) { + if (diffnorm > clip_gradients) { clip_scale = clip_gradients / diffnorm; } } - // iterate over active params for (set::iterator it = active_params_set_.begin(); it != active_params_set_.end(); ++it) { const string& param_name = *it; shared_ptr > curParam = params_[param_name]; + Dtype lrNew = lr * clip_scale * param_lr_mults()[param_name]; const shared_ptr > &curData = curParam->data(); const shared_ptr > &curDiff = curParam->diff(); curDiff->AddMulFrom(*curData, weight_decay); - curData->AddMulFrom(*curDiff, -lr); + curData->AddMulFrom(*curDiff, -lrNew); curParam->scale_diff(momentum); } } From ebbb6ed9d56125f52d607746ccdb433e448292f4 Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sun, 16 Aug 2015 09:57:06 -0400 Subject: [PATCH 8/9] fixed lint errors --- include/caffe/apollonet.hpp | 3 ++- include/caffe/data_layers.hpp | 6 +++--- include/caffe/layer_factory.hpp | 8 ++++---- scripts/cpp_lint.py | 2 ++ src/caffe/layers/mat_data_layer.cpp | 9 ++++----- 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/include/caffe/apollonet.hpp b/include/caffe/apollonet.hpp index f7c166b46b4..d8b9758c0b5 100644 --- a/include/caffe/apollonet.hpp +++ b/include/caffe/apollonet.hpp @@ -52,7 +52,8 @@ class ApolloNet { CopyTrainedLayersFrom(param); } - void Update(Dtype lr, Dtype momentum, Dtype clip_gradients, Dtype weight_decay); + void Update(Dtype lr, Dtype momentum, Dtype clip_gradients, + Dtype weight_decay); /// @brief returns the phase: TRAIN or TEST inline Phase phase() const { return phase_; } diff --git a/include/caffe/data_layers.hpp b/include/caffe/data_layers.hpp index 3b552b3de7a..122c8bec28e 100644 --- a/include/caffe/data_layers.hpp +++ b/include/caffe/data_layers.hpp @@ -318,9 +318,9 @@ class MatDataLayer : public BaseDataLayer { protected: virtual void Forward_cpu(const vector*>& bottom, const vector*>& top); - - shared_ptr > data_transformer(){ - if (!this->data_transformer_){ + + shared_ptr > data_transformer() { + if (!this->data_transformer_) { this->data_transformer_.reset( new DataTransformer(this->transform_param_, this->phase_)); } diff --git a/include/caffe/layer_factory.hpp b/include/caffe/layer_factory.hpp index 1b1764f21ca..dfe818a9610 100644 --- a/include/caffe/layer_factory.hpp +++ b/include/caffe/layer_factory.hpp @@ -80,13 +80,13 @@ class LayerRegistry { << " (known types: " << LayerTypeList() << ")"; return registry[type](param); } - + // Get a layer using a prototxt which will be parsed into LayerParameter static shared_ptr > CreateLayer(const string& prototxt) { - LayerParameter param; - bool success = google::protobuf::TextFormat::ParseFromString(prototxt, ¶m); + LayerParameter p; + bool success = google::protobuf::TextFormat::ParseFromString(prototxt, &p); ASSERT(success, "Invalid prototxt string"); - return CreateLayer(param); + return CreateLayer(p); } private: diff --git a/scripts/cpp_lint.py b/scripts/cpp_lint.py index f750489f4f9..57ba160144a 100755 --- a/scripts/cpp_lint.py +++ b/scripts/cpp_lint.py @@ -1610,6 +1610,7 @@ def CheckCaffeDataLayerSetUp(filename, clean_lines, linenum, error): if ix >= 0 and ( line.find('void DataLayer::LayerSetUp') != -1 or line.find('void ImageDataLayer::LayerSetUp') != -1 or + line.find('void MatDataLayer::LayerSetUp') != -1 or line.find('void MemoryDataLayer::LayerSetUp') != -1 or line.find('void WindowDataLayer::LayerSetUp') != -1): error(filename, linenum, 'caffe/data_layer_setup', 2, @@ -1622,6 +1623,7 @@ def CheckCaffeDataLayerSetUp(filename, clean_lines, linenum, error): line.find('void Base') == -1 and line.find('void DataLayer::DataLayerSetUp') == -1 and line.find('void ImageDataLayer::DataLayerSetUp') == -1 and + line.find('void MatDataLayer::DataLayerSetUp') == -1 and line.find('void MemoryDataLayer::DataLayerSetUp') == -1 and line.find('void WindowDataLayer::DataLayerSetUp') == -1): error(filename, linenum, 'caffe/data_layer_setup', 2, diff --git a/src/caffe/layers/mat_data_layer.cpp b/src/caffe/layers/mat_data_layer.cpp index 21463531618..9f2f0d82ab2 100644 --- a/src/caffe/layers/mat_data_layer.cpp +++ b/src/caffe/layers/mat_data_layer.cpp @@ -14,19 +14,18 @@ void MatDataLayer::DataLayerSetUp(const vector*>& bottom, template void MatDataLayer::AddMatVector(const vector& mat_vector) { - // reshape layers + // reshape layers batch_size_ = mat_vector.size(); channels_ = mat_vector[0].channels(); height_ = mat_vector[0].rows; width_ = mat_vector[0].cols; size_ = channels_ * height_ * width_; CHECK_GT(batch_size_ * size_, 0) << - "batch_size, channels, height, and width must be specified and" - " positive in memory_data_param"; + "batch_size, channels, height, and width must be positive"; added_data_.Reshape(batch_size_, channels_, height_, width_); - //TODO: is this necessary + // TODO: is this necessary added_data_.cpu_data(); - + // Apply data transformations (mirror, scale, crop...) data_transformer()->Transform(mat_vector, &added_data_); Dtype* top_data = added_data_.mutable_cpu_data(); From 903b13a9b3e66d7d74b89a343f03fc2d136b99e4 Mon Sep 17 00:00:00 2001 From: nicodjimenez Date: Sun, 16 Aug 2015 10:57:50 -0400 Subject: [PATCH 9/9] added decay mult and changed from camel to underscore --- src/caffe/apollonet.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/caffe/apollonet.cpp b/src/caffe/apollonet.cpp index f9fdf4df7c4..d76ba48cba0 100644 --- a/src/caffe/apollonet.cpp +++ b/src/caffe/apollonet.cpp @@ -339,13 +339,14 @@ void ApolloNet::Update(Dtype lr, Dtype momentum, for (set::iterator it = active_params_set_.begin(); it != active_params_set_.end(); ++it) { const string& param_name = *it; - shared_ptr > curParam = params_[param_name]; - Dtype lrNew = lr * clip_scale * param_lr_mults()[param_name]; - const shared_ptr > &curData = curParam->data(); - const shared_ptr > &curDiff = curParam->diff(); - curDiff->AddMulFrom(*curData, weight_decay); - curData->AddMulFrom(*curDiff, -lrNew); - curParam->scale_diff(momentum); + shared_ptr > cur_param = params_[param_name]; + Dtype lr_new = lr * clip_scale * param_lr_mults()[param_name]; + Dtype decay_new = weight_decay * param_decay_mults()[param_name]; + const shared_ptr > &cur_data = cur_param->data(); + const shared_ptr > &cur_diff = cur_param->diff(); + cur_diff->AddMulFrom(*cur_data, decay_new); + cur_data->AddMulFrom(*cur_diff, -lr_new); + cur_param->scale_diff(momentum); } }