diff --git a/docs/feature_extraction.md b/docs/feature_extraction.md new file mode 100644 index 00000000000..2fb5d19d57d --- /dev/null +++ b/docs/feature_extraction.md @@ -0,0 +1,61 @@ +--- +layout: default +title: Caffe +--- + +Extracting Features Using Pre-trained Model +=========================================== + +CAFFE represents Convolution Architecture For Feature Extraction. Extracting features using pre-trained model is one of the strongest requirements users ask for. + +Because of the record-breaking image classification accuracy and the flexible domain adaptability of [the network architecture proposed by Krizhevsky, Sutskever, and Hinton](http://books.nips.cc/papers/files/nips25/NIPS2012_0534.pdf), Caffe provides a pre-trained reference image model to save you from days of training. + +If you need detailed usage help information of the involved tools, please read the source code of them which provide everything you need to know about. + +Get the Reference Model +----------------------- + +Assume you are in the root directory of Caffe. + + cd models + ./get_caffe_reference_imagenet_model.sh + +After the downloading is finished, you will have models/caffe_reference_imagenet_model. + +Preprocess the Data +------------------- + +Generate a list of the files to process. + + examples/feature_extraction/generate_file_list.py /your/images/dir /your/images.txt + +The network definition of the reference model only accepts 256*256 pixel images stored in the leveldb format. First, resize your images if they do not match the required size. + + build/tools/resize_and_crop_images.py --num_clients=8 --image_lib=opencv --output_side_length=256 --input=/your/images.txt --input_folder=/your/images/dir --output_folder=/your/resized/images/dir_256_256 + +Set the num_clients to be the number of CPU cores on your machine. Run "nproc" or "cat /proc/cpuinfo | grep processor | wc -l" to get the number on Linux. + + build/tools/generate_file_list.py /your/resized/images/dir_256_256 /your/resized/images_256_256.txt + build/tools/convert_imageset /your/resized/images/dir_256_256 /your/resized/images_256_256.txt /your/resized/images_256_256_leveldb 1 + +In practice, subtracting the mean image from a dataset significantly improves classification accuracies. Download the mean image of the ILSVRC dataset. + + data/ilsvrc12/get_ilsvrc_aux.sh + +You can directly use the imagenet_mean.binaryproto in the network definition proto. If you have a large number of images, you can also compute the mean of all the images. + + build/tools/compute_image_mean.bin /your/resized/images_256_256_leveldb /your/resized/images_256_256_mean.binaryproto + +Define the Feature Extraction Network Architecture +-------------------------------------------------- + +If you do not want to change the reference model network architecture , simply copy examples/imagenet into examples/your_own_dir. Then point the source and meanfile field of the data layer in imagenet_val.prototxt to /your/resized/images_256_256_leveldb and /your/resized/images_256_256_mean.binaryproto respectively. + +Extract Features +---------------- + +Now everything necessary is in place. + + build/tools/extract_features.bin models/caffe_reference_imagenet_model examples/feature_extraction/imagenet_val.prototxt fc7 examples/feature_extraction/features 10 + +The name of feature blob that you extract is fc7 which represents the highest level feature of the reference model. Any other blob is also applicable. The last parameter above is the number of data mini-batches. diff --git a/examples/feature_extraction/generate_file_list.py b/examples/feature_extraction/generate_file_list.py new file mode 100755 index 00000000000..c0dcb938893 --- /dev/null +++ b/examples/feature_extraction/generate_file_list.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +import os +import sys + +def help(): + print 'Usage: ./generate_file_list.py file_dir file_list.txt' + exit(1) + +def main(): + if len(sys.argv) < 3: + help() + file_dir = sys.argv[1] + file_list_txt = sys.argv[2] + if not os.path.exists(file_dir): + print 'Error: file dir does not exist ', file_dir + exit(1) + file_dir = os.path.abspath(file_dir) + '/' + with open(file_list_txt, 'w') as output: + for root, dirs, files in os.walk(file_dir): + for name in files: + file_path = file_path.replace(os.path.join(root, name), '') + output.write(file_path + '\n') + +if __name__ == '__main__': + main() diff --git a/include/caffe/net.hpp b/include/caffe/net.hpp index b5a57b3c5a4..c279ba698aa 100644 --- a/include/caffe/net.hpp +++ b/include/caffe/net.hpp @@ -82,6 +82,13 @@ class Net { inline int num_outputs() { return net_output_blobs_.size(); } inline vector*>& input_blobs() { return net_input_blobs_; } inline vector*>& output_blobs() { return net_output_blobs_; } + // has_blob and blob_by_name are inspired by + // https://github.com/kencoken/caffe/commit/f36e71569455c9fbb4bf8a63c2d53224e32a4e7b + // Access intermediary computation layers, testing with centre image only + bool has_blob(const string& blob_name); + const shared_ptr > blob_by_name(const string& blob_name); + bool has_layer(const string& layer_name); + const shared_ptr > layer_by_name(const string& layer_name); protected: // Function to get misc parameters, e.g. the learning rate multiplier and @@ -91,11 +98,13 @@ class Net { // Individual layers in the net vector > > layers_; vector layer_names_; + map layer_names_index_; vector layer_need_backward_; // blobs stores the blobs that store intermediate results between the // layers. vector > > blobs_; vector blob_names_; + map blob_names_index_; vector blob_need_backward_; // bottom_vecs stores the vectors containing the input for each layer. // They don't actually host the blobs (blobs_ does), so we simply store diff --git a/include/caffe/util/math_functions.hpp b/include/caffe/util/math_functions.hpp index e9e2db8f274..26abb2d02c2 100644 --- a/include/caffe/util/math_functions.hpp +++ b/include/caffe/util/math_functions.hpp @@ -1,4 +1,5 @@ // Copyright 2013 Yangqing Jia +// Copyright 2014 kloudkl@github #ifndef CAFFE_UTIL_MATH_FUNCTIONS_H_ #define CAFFE_UTIL_MATH_FUNCTIONS_H_ @@ -100,6 +101,9 @@ Dtype caffe_cpu_dot(const int n, const Dtype* x, const Dtype* y); template void caffe_gpu_dot(const int n, const Dtype* x, const Dtype* y, Dtype* out); +template +int caffe_hamming_distance(const int n, const Dtype* x, const Dtype* y); + } // namespace caffe diff --git a/src/caffe/net.cpp b/src/caffe/net.cpp index 1837b0768ae..c979a967010 100644 --- a/src/caffe/net.cpp +++ b/src/caffe/net.cpp @@ -162,6 +162,12 @@ void Net::Init(const NetParameter& in_param) { LOG(INFO) << "This network produces output " << *it; net_output_blobs_.push_back(blobs_[blob_name_to_idx[*it]].get()); } + for (size_t i = 0; i < blob_names_.size(); ++i) { + blob_names_index_[blob_names_[i]] = i; + } + for (size_t i = 0; i < layer_names_.size(); ++i) { + layer_names_index_[layer_names_[i]] = i; + } GetLearningRateAndWeightDecay(); LOG(INFO) << "Network initialization done."; LOG(INFO) << "Memory required for Data " << memory_used*sizeof(Dtype); @@ -327,6 +333,42 @@ void Net::Update() { } } +template +bool Net::has_blob(const string& blob_name) { + return blob_names_index_.find(blob_name) != blob_names_index_.end(); +} + +template +const shared_ptr > Net::blob_by_name( + const string& blob_name) { + shared_ptr > blob_ptr; + if (has_blob(blob_name)) { + blob_ptr = blobs_[blob_names_index_[blob_name]]; + } else { + blob_ptr.reset((Blob*)(NULL)); + LOG(WARNING) << "Unknown blob name " << blob_name; + } + return blob_ptr; +} + +template +bool Net::has_layer(const string& layer_name) { + return layer_names_index_.find(layer_name) != layer_names_index_.end(); +} + +template +const shared_ptr > Net::layer_by_name( + const string& layer_name) { + shared_ptr > layer_ptr; + if (has_layer(layer_name)) { + layer_ptr = layers_[layer_names_index_[layer_name]]; + } else { + layer_ptr.reset((Layer*)(NULL)); + LOG(WARNING) << "Unknown layer name " << layer_name; + } + return layer_ptr; +} + INSTANTIATE_CLASS(Net); } // namespace caffe diff --git a/src/caffe/test/test_math_functions.cpp b/src/caffe/test/test_math_functions.cpp new file mode 100644 index 00000000000..45d43cc9415 --- /dev/null +++ b/src/caffe/test/test_math_functions.cpp @@ -0,0 +1,77 @@ +// Copyright 2014 kloudkl@github + +#include // for uint32_t & uint64_t + +#include "gtest/gtest.h" +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/filler.hpp" +#include "caffe/util/math_functions.hpp" + +#include "caffe/test/test_caffe_main.hpp" + +namespace caffe { + +template +class MathFunctionsTest : public ::testing::Test { + protected: + MathFunctionsTest() + : blob_bottom_(new Blob()), + blob_top_(new Blob()) { + } + + virtual void SetUp() { + Caffe::set_random_seed(1701); + this->blob_bottom_->Reshape(100, 70, 50, 30); + this->blob_top_->Reshape(100, 70, 50, 30); + // fill the values + FillerParameter filler_param; + GaussianFiller filler(filler_param); + filler.Fill(this->blob_bottom_); + filler.Fill(this->blob_top_); + } + + virtual ~MathFunctionsTest() { + delete blob_bottom_; + delete blob_top_; + } + // http://en.wikipedia.org/wiki/Hamming_distance + int ReferenceHammingDistance(const int n, const Dtype* x, const Dtype* y); + + Blob* const blob_bottom_; + Blob* const blob_top_; +}; + +#define REF_HAMMING_DIST(float_type, int_type) \ +template<> \ +int MathFunctionsTest::ReferenceHammingDistance(const int n, \ + const float_type* x, \ + const float_type* y) { \ + int dist = 0; \ + int_type val; \ + for (int i = 0; i < n; ++i) { \ + val = static_cast(x[i]) ^ static_cast(y[i]); \ + /* Count the number of set bits */ \ + while (val) { \ + ++dist; \ + val &= val - 1; \ + } \ + } \ + return dist; \ +} + +REF_HAMMING_DIST(float, uint32_t); +REF_HAMMING_DIST(double, uint64_t); + +typedef ::testing::Types Dtypes; +TYPED_TEST_CASE(MathFunctionsTest, Dtypes); + +TYPED_TEST(MathFunctionsTest, TestHammingDistance) { + int n = this->blob_bottom_->count(); + const TypeParam* x = this->blob_bottom_->cpu_data(); + const TypeParam* y = this->blob_top_->cpu_data(); + CHECK_EQ(this->ReferenceHammingDistance(n, x, y), + caffe_hamming_distance(n, x, y)); +} + +} // namespace caffe diff --git a/src/caffe/test/test_net.cpp b/src/caffe/test/test_net.cpp new file mode 100644 index 00000000000..fd7265c47df --- /dev/null +++ b/src/caffe/test/test_net.cpp @@ -0,0 +1,148 @@ +// Copyright 2014 kloudkl@github + +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "caffe/common.hpp" +#include "caffe/net.hpp" +#include "caffe/test/test_gradient_check_util.hpp" + +#include "caffe/test/test_caffe_main.hpp" + +namespace caffe { + + +template +class NetTest : public ::testing::Test { + protected: + NetTest() : filename(NULL) { + } + + virtual void SetUp() { // Create the leveldb + filename = tmpnam(NULL); // get temp name + LOG(INFO) << "Using temporary leveldb " << filename; + leveldb::DB* db; + leveldb::Options options; + options.error_if_exists = true; + options.create_if_missing = true; + leveldb::Status status = leveldb::DB::Open(options, filename, &db); + CHECK(status.ok()); + for (int i = 0; i < 5; ++i) { + Datum datum; + datum.set_label(i); + datum.set_channels(2); + datum.set_height(3); + datum.set_width(4); + std::string* data = datum.mutable_data(); + for (int j = 0; j < 24; ++j) { + data->push_back((uint8_t)i); + } + std::stringstream ss; + ss << i; + db->Put(leveldb::WriteOptions(), ss.str(), datum.SerializeAsString()); + } + delete db; + + const string& proto_prefix = + "name: 'TestNetwork' " + "layers: { " + " layer { " + " name: 'data' " + " type: 'data' "; + const string& proto_suffix = + " batchsize: 1 " + " } " + " top: 'data' " + " top: 'label' " + "} " + "layers: { " + " layer { " + " name: 'innerproduct' " + " type: 'innerproduct' " + " num_output: 1000 " + " weight_filler { " + " type: 'gaussian' " + " std: 0.01 " + " } " + " bias_filler { " + " type: 'constant' " + " value: 0 " + " } " + " blobs_lr: 1. " + " blobs_lr: 2. " + " weight_decay: 1. " + " weight_decay: 0. " + " } " + " bottom: 'data' " + " top: 'innerproduct' " + "} " + "layers: { " + " layer { " + " name: 'loss' " + " type: 'softmax_loss' " + " } " + " bottom: 'innerproduct' " + " bottom: 'label' " + "} "; + proto = proto_prefix + "source: '" + string(this->filename) + + "' " + proto_suffix; + } + + virtual ~NetTest() { + } + + char* filename; + string proto; +}; + +typedef ::testing::Types Dtypes; +TYPED_TEST_CASE(NetTest, Dtypes); + +TYPED_TEST(NetTest, TestHasBlob) { + NetParameter param; + CHECK(google::protobuf::TextFormat::ParseFromString(this->proto, + ¶m)); + Net net(param); + EXPECT_TRUE(net.has_blob("data")); + EXPECT_TRUE(net.has_blob("label")); + EXPECT_TRUE(net.has_blob("innerproduct")); + EXPECT_FALSE(net.has_blob("loss")); +} + +TYPED_TEST(NetTest, TestGetBlob) { + NetParameter param; + CHECK(google::protobuf::TextFormat::ParseFromString(this->proto, + ¶m)); + Net net(param); + EXPECT_EQ(net.blob_by_name("data"), net.blobs()[0]); + EXPECT_EQ(net.blob_by_name("label"), net.blobs()[1]); + EXPECT_EQ(net.blob_by_name("innerproduct"), net.blobs()[2]); + EXPECT_FALSE(net.blob_by_name("loss")); +} + +TYPED_TEST(NetTest, TestHasLayer) { + NetParameter param; + CHECK(google::protobuf::TextFormat::ParseFromString(this->proto, + ¶m)); + Net net(param); + EXPECT_TRUE(net.has_layer("data")); + EXPECT_TRUE(net.has_layer("innerproduct")); + EXPECT_TRUE(net.has_layer("loss")); + EXPECT_FALSE(net.has_layer("label")); +} + +TYPED_TEST(NetTest, TestGetLayerByName) { + NetParameter param; + CHECK(google::protobuf::TextFormat::ParseFromString(this->proto, + ¶m)); + Net net(param); + EXPECT_EQ(net.layer_by_name("data"), net.layers()[0]); + EXPECT_EQ(net.layer_by_name("innerproduct"), net.layers()[1]); + EXPECT_EQ(net.layer_by_name("loss"), net.layers()[2]); + EXPECT_FALSE(net.layer_by_name("label")); +} + +} // namespace caffe diff --git a/src/caffe/util/math_functions.cpp b/src/caffe/util/math_functions.cpp index 60656b87093..790f00eaf0e 100644 --- a/src/caffe/util/math_functions.cpp +++ b/src/caffe/util/math_functions.cpp @@ -1,4 +1,5 @@ // Copyright 2013 Yangqing Jia +// Copyright 2014 kloudkl@github #include #include @@ -293,4 +294,26 @@ void caffe_gpu_dot(const int n, const double* x, const double* y, CUBLAS_CHECK(cublasDdot(Caffe::cublas_handle(), n, x, 1, y, 1, out)); } +template <> +int caffe_hamming_distance(const int n, const float* x, + const float* y) { + int dist = 0; + for (int i = 0; i < n; ++i) { + dist += __builtin_popcount(static_cast(x[i]) ^ + static_cast(y[i])); + } + return dist; +} + +template <> +int caffe_hamming_distance(const int n, const double* x, + const double* y) { + int dist = 0; + for (int i = 0; i < n; ++i) { + dist += __builtin_popcountl(static_cast(x[i]) ^ + static_cast(y[i])); + } + return dist; +} + } // namespace caffe diff --git a/tools/extract_features.cpp b/tools/extract_features.cpp new file mode 100644 index 00000000000..e547db594ba --- /dev/null +++ b/tools/extract_features.cpp @@ -0,0 +1,173 @@ +// Copyright 2014 kloudkl@github + +#include // for snprintf +#include +#include +#include +#include +#include +#include + +#include "caffe/blob.hpp" +#include "caffe/common.hpp" +#include "caffe/net.hpp" +#include "caffe/vision_layers.hpp" +#include "caffe/proto/caffe.pb.h" +#include "caffe/util/io.hpp" + +using namespace caffe; // NOLINT(build/namespaces) + +template +int feature_extraction_pipeline(int argc, char** argv); + +int main(int argc, char** argv) { + return feature_extraction_pipeline(argc, argv); +// return feature_extraction_pipeline(argc, argv); +} + +template +int feature_extraction_pipeline(int argc, char** argv) { + const int num_required_args = 6; + if (argc < num_required_args) { + LOG(ERROR)<< + "This program takes in a trained network and an input data layer, and then" + " extract features of the input data produced by the net.\n" + "Usage: demo_extract_features pretrained_net_param" + " feature_extraction_proto_file extract_feature_blob_name" + " save_feature_leveldb_name num_mini_batches [CPU/GPU] [DEVICE_ID=0]"; + return 1; + } + int arg_pos = num_required_args; + + arg_pos = num_required_args; + if (argc > arg_pos && strcmp(argv[arg_pos], "GPU") == 0) { + LOG(ERROR)<< "Using GPU"; + uint device_id = 0; + if (argc > arg_pos + 1) { + device_id = atoi(argv[arg_pos + 1]); + CHECK_GE(device_id, 0); + } + LOG(ERROR) << "Using Device_id=" << device_id; + Caffe::SetDevice(device_id); + Caffe::set_mode(Caffe::GPU); + } else { + LOG(ERROR) << "Using CPU"; + Caffe::set_mode(Caffe::CPU); + } + Caffe::set_phase(Caffe::TEST); + + NetParameter pretrained_net_param; + + arg_pos = 0; // the name of the executable + string pretrained_binary_proto(argv[++arg_pos]); + ReadProtoFromBinaryFile(pretrained_binary_proto.c_str(), + &pretrained_net_param); + + // Expected prototxt contains at least one data layer such as + // the layer data_layer_name and one feature blob such as the + // fc7 top blob to extract features. + /* + layers { + layer { + name: "data_layer_name" + type: "data" + source: "/path/to/your/images/to/extract/feature/images_leveldb" + meanfile: "/path/to/your/image_mean.binaryproto" + batchsize: 128 + cropsize: 227 + mirror: false + } + top: "data_blob_name" + top: "label_blob_name" + } + layers { + layer { + name: "drop7" + type: "dropout" + dropout_ratio: 0.5 + } + bottom: "fc7" + top: "fc7" + } + */ + NetParameter feature_extraction_net_param; + string feature_extraction_proto(argv[++arg_pos]); + ReadProtoFromTextFile(feature_extraction_proto, + &feature_extraction_net_param); + shared_ptr > feature_extraction_net( + new Net(feature_extraction_net_param)); + feature_extraction_net->CopyTrainedLayersFrom(pretrained_net_param); + + string extract_feature_blob_name(argv[++arg_pos]); + CHECK(feature_extraction_net->has_blob(extract_feature_blob_name)) + << "Unknown feature blob name " << extract_feature_blob_name + << " in the network " << feature_extraction_proto; + + string save_feature_leveldb_name(argv[++arg_pos]); + leveldb::DB* db; + leveldb::Options options; + options.error_if_exists = true; + options.create_if_missing = true; + options.write_buffer_size = 268435456; + LOG(INFO)<< "Opening leveldb " << save_feature_leveldb_name; + leveldb::Status status = leveldb::DB::Open(options, + save_feature_leveldb_name.c_str(), + &db); + CHECK(status.ok()) << "Failed to open leveldb " << save_feature_leveldb_name; + + int num_mini_batches = atoi(argv[++arg_pos]); + + LOG(ERROR)<< "Extacting Features"; + + Datum datum; + leveldb::WriteBatch* batch = new leveldb::WriteBatch(); + const int kMaxKeyStrLength = 100; + char key_str[kMaxKeyStrLength]; + int num_bytes_of_binary_code = sizeof(Dtype); + vector*> input_vec; + int image_index = 0; + for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index) { + feature_extraction_net->Forward(input_vec); + const shared_ptr > feature_blob = feature_extraction_net + ->blob_by_name(extract_feature_blob_name); + int num_features = feature_blob->num(); + int dim_features = feature_blob->count() / num_features; + Dtype* feature_blob_data; + for (int n = 0; n < num_features; ++n) { + datum.set_height(dim_features); + datum.set_width(1); + datum.set_channels(1); + datum.clear_data(); + datum.clear_float_data(); + feature_blob_data = feature_blob->mutable_cpu_data() + + feature_blob->offset(n); + for (int d = 0; d < dim_features; ++d) { + datum.add_float_data(feature_blob_data[d]); + } + string value; + datum.SerializeToString(&value); + snprintf(key_str, kMaxKeyStrLength, "%d", image_index); + batch->Put(string(key_str), value); + ++image_index; + if (image_index % 1000 == 0) { + db->Write(leveldb::WriteOptions(), batch); + LOG(ERROR)<< "Extracted features of " << image_index << + " query images."; + delete batch; + batch = new leveldb::WriteBatch(); + } + } // for (int n = 0; n < num_features; ++n) + } // for (int batch_index = 0; batch_index < num_mini_batches; ++batch_index) + // write the last batch + if (image_index % 1000 != 0) { + db->Write(leveldb::WriteOptions(), batch); + LOG(ERROR)<< "Extracted features of " << image_index << + " query images."; + } + + delete batch; + delete db; + LOG(ERROR)<< "Successfully extracted the features!"; + return 0; +} +