This repository has been archived by the owner on Nov 17, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6.8k
Local IO #28
Merged
Merged
Local IO #28
Changes from 6 commits
Commits
Show all changes
65 commits
Select commit
Hold shift + click to select a range
74c41d7
finish merge remote master
sneakerkg 63ff6d0
built in python, start polishing new feature required
sneakerkg 065b061
finish old version registry in C
sneakerkg 03f05d4
modify to dmlc registry
sneakerkg 7144cb4
pass python mnist test, begin cleaning
sneakerkg 4f6ef5d
clean io interface
sneakerkg 12b3d22
finish merge remote master
sneakerkg f673f7d
built in python, start polishing new feature required
sneakerkg a8dcbd1
finish old version registry in C
sneakerkg 7f36bbb
modify to dmlc registry
sneakerkg 299e1d8
pass python mnist test, begin cleaning
sneakerkg 7cf5506
clean io interface
sneakerkg 994bc41
modify to pass travis
sneakerkg 819e5f8
merging latest master
sneakerkg 3e406ff
finish merge remote master
sneakerkg c81b60b
built in python, start polishing new feature required
sneakerkg 0ebf6aa
finish old version registry in C
sneakerkg 7511c12
modify to dmlc registry
sneakerkg c03d62b
pass python mnist test, begin cleaning
sneakerkg 08dcaf7
clean io interface
sneakerkg bcd2652
finish merge remote master
sneakerkg 97a88cb
built in python, start polishing new feature required
sneakerkg 6915299
finish old version registry in C
sneakerkg 305d2af
modify to dmlc registry
sneakerkg 04f9888
pass python mnist test, begin cleaning
sneakerkg c6ae534
clean io interface
sneakerkg 3da4ac0
modify to pass travis
sneakerkg f4fd400
finish refactoring io code
sneakerkg b6ebf94
add io.md in doc
sneakerkg 6e1b7dd
merged latest master
sneakerkg 8021478
resolve rebase problem
sneakerkg 2ad67a3
Merge pull request #34 from antinucleon/master
antinucleon 1b19f9d
finish merge remote master
sneakerkg 0c24dbf
built in python, start polishing new feature required
sneakerkg a2f03a7
finish old version registry in C
sneakerkg 9d4d98e
modify to dmlc registry
sneakerkg 1544258
pass python mnist test, begin cleaning
sneakerkg 36cff22
clean io interface
sneakerkg 3ee7f24
finish merge remote master
sneakerkg 88d2db7
built in python, start polishing new feature required
sneakerkg 2ff4ff0
finish old version registry in C
sneakerkg 9b002de
modify to dmlc registry
sneakerkg c29793f
pass python mnist test, begin cleaning
sneakerkg 56b4129
clean io interface
sneakerkg e48e400
modify to pass travis
sneakerkg 49728eb
finish merge remote master
sneakerkg cb99a1e
built in python, start polishing new feature required
sneakerkg 24ad6f5
finish old version registry in C
sneakerkg 3931343
modify to dmlc registry
sneakerkg b1e2bd9
pass python mnist test, begin cleaning
sneakerkg 0e99411
clean io interface
sneakerkg 51b7208
finish merge remote master
sneakerkg 358a623
finish old version registry in C
sneakerkg 94c6872
modify to dmlc registry
sneakerkg 8caa4ac
pass python mnist test, begin cleaning
sneakerkg 3fc60b1
clean io interface
sneakerkg af12892
finish refactoring io code
sneakerkg 08f12b8
add io.md in doc
sneakerkg 9bdcebd
merged latest master
sneakerkg e68452f
add local io into test
sneakerkg 32d10ae
use local io in test
sneakerkg a4e70b2
merge master
sneakerkg f1b9764
meet python3 strict
sneakerkg 982f18d
can not train in python3
sneakerkg 25e3363
list zip for py3, no reset in iter
sneakerkg File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,115 @@ | ||
/*! | ||
* Copyright (c) 2015 by Contributors | ||
* \file io.h | ||
* \brief mxnet io data structure and data iterator | ||
*/ | ||
#ifndef MXNET_IO_H_ | ||
#define MXNET_IO_H_ | ||
#include <dmlc/data.h> | ||
#include <dmlc/registry.h> | ||
#include <vector> | ||
#include <string> | ||
#include <utility> | ||
#include "./base.h" | ||
|
||
namespace mxnet { | ||
/*! | ||
* \brief iterator type | ||
* \tparam DType data type | ||
*/ | ||
template<typename DType> | ||
class IIterator : public dmlc::DataIter<DType> { | ||
public: | ||
/*! | ||
* \brief set the parameters and init iter | ||
* \param kwargs key-value pairs | ||
*/ | ||
virtual void SetInit(const std::vector<std::pair<std::string, std::string> >& kwargs) = 0; | ||
/*! \brief initalize the iterator so that we can use the iterator */ | ||
virtual void Init(void) = 0; | ||
/*! \brief set before first of the item */ | ||
virtual void BeforeFirst(void) = 0; | ||
/*! \brief move to next item */ | ||
virtual bool Next(void) = 0; | ||
/*! \brief get current data */ | ||
virtual const DType &Value(void) const = 0; | ||
/*! \brief constructor */ | ||
virtual ~IIterator(void) {} | ||
/*! \brief store the name of each data, it could be used for making NArrays */ | ||
std::vector<std::string> data_names; | ||
/*! \brief set data name to each attribute of data */ | ||
inline void SetDataName(const std::string data_name){ | ||
data_names.push_back(data_name); | ||
} | ||
}; // class IIterator | ||
|
||
/*! \brief a single data instance */ | ||
struct DataInst { | ||
/*! \brief unique id for instance */ | ||
unsigned index; | ||
/*! \brief content of data */ | ||
std::vector<TBlob> data; | ||
/*! \brief extra data to be fed to the network */ | ||
std::string extra_data; | ||
}; // struct DataInst | ||
|
||
/*! | ||
* \brief a standard batch of data commonly used by iterator | ||
* a databatch contains multiple TBlobs. Each Tblobs has | ||
* a name stored in a map. There's no different between | ||
* data and label, how we use them is to see the DNN implementation. | ||
*/ | ||
struct DataBatch { | ||
public: | ||
/*! \brief unique id for instance, can be NULL, sometimes is useful */ | ||
unsigned *inst_index; | ||
/*! \brief number of instance */ | ||
mshadow::index_t batch_size; | ||
/*! \brief number of padding elements in this batch, | ||
this is used to indicate the last elements in the batch are only padded up to match the batch, and should be discarded */ | ||
mshadow::index_t num_batch_padd; | ||
public: | ||
/*! \brief content of dense data, if this DataBatch is dense */ | ||
std::vector<TBlob> data; | ||
/*! \brief extra data to be fed to the network */ | ||
std::string extra_data; | ||
public: | ||
/*! \brief constructor */ | ||
DataBatch(void) { | ||
inst_index = NULL; | ||
batch_size = 0; num_batch_padd = 0; | ||
} | ||
/*! \brief giving name to the data */ | ||
void Naming(std::vector<std::string> names); | ||
}; // struct DataBatch | ||
|
||
/*! \brief typedef the factory function of data iterator */ | ||
typedef IIterator<DataBatch> *(*DataIteratorFactory)(); | ||
/*! | ||
* \brief Registry entry for DataIterator factory functions. | ||
*/ | ||
struct DataIteratorReg | ||
: public dmlc::FunctionRegEntryBase<DataIteratorReg, | ||
DataIteratorFactory> { | ||
}; | ||
//-------------------------------------------------------------- | ||
// The following part are API Registration of Iterators | ||
//-------------------------------------------------------------- | ||
/*! | ||
* \brief Macro to register Iterators | ||
* | ||
* \code | ||
* // example of registering a mnist iterator | ||
* REGISTER_IO_ITERATOR(MNIST, MNISTIterator) | ||
* .describe("Mnist data iterator"); | ||
* | ||
* \endcode | ||
*/ | ||
#define MXNET_REGISTER_IO_ITER(name, DataIteratorType) \ | ||
static ::mxnet::IIterator<DataBatch>* __create__ ## DataIteratorType ## __() { \ | ||
return new DataIteratorType; \ | ||
} \ | ||
DMLC_REGISTRY_REGISTER(::mxnet::DataIteratorReg, DataIteratorReg, name) \ | ||
.set_body(__create__ ## DataIteratorType ## __) | ||
} // namespace mxnet | ||
#endif // MXNET_IO_H_ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
variable name, should be like data_