From e00bc8f928214e961da83f0c2dd5d8d62d9557d7 Mon Sep 17 00:00:00 2001 From: "Wang,Jeff" Date: Thu, 8 Mar 2018 16:17:05 -0800 Subject: [PATCH 1/3] Only write the modified tablets to file system. Move some functions definitions from headers to cc files. --- visualdl/storage/storage.cc | 33 ++++++++++++++++++++++++++++++--- visualdl/storage/storage.h | 14 +++++++------- visualdl/storage/tablet.cc | 16 ++++++++++++++++ visualdl/storage/tablet.h | 20 ++++++-------------- 4 files changed, 59 insertions(+), 24 deletions(-) diff --git a/visualdl/storage/storage.cc b/visualdl/storage/storage.cc index f7a5bee15..5f4b3e7b8 100644 --- a/visualdl/storage/storage.cc +++ b/visualdl/storage/storage.cc @@ -21,6 +21,7 @@ Storage::Storage() { data_ = std::make_shared(); tablets_ = std::make_shared>(); modes_ = std::make_shared>(); + modified_tablet_set_ = std::set(); time_t t; time(&t); data_->set_timestamp(t); @@ -43,11 +44,20 @@ Tablet Storage::AddTablet(const std::string& x) { CHECK(tablets_->count(x) == 0) << "tablet [" << x << "] has existed"; (*tablets_)[x] = storage::Tablet(); AddTag(x); + MarkTabletModified(x); // WRITE_GUARD PersistToDisk(); return Tablet(&(*tablets_)[x], this); } +void Storage::SetDir(const std::string& dir) { + *dir_ = dir; +} + +std::string Storage::dir() const { + return *dir_; +} + void Storage::PersistToDisk() { PersistToDisk(*dir_); } void Storage::PersistToDisk(const std::string& dir) { @@ -56,12 +66,29 @@ void Storage::PersistToDisk(const std::string& dir) { fs::SerializeToFile(*data_, meta_path(dir)); for (auto tag : data_->tags()) { - auto it = tablets_->find(tag); - CHECK(it != tablets_->end()) << "tag " << tag << " not exist."; - fs::SerializeToFile(it->second, tablet_path(dir, tag)); + if (modified_tablet_set_.count(tag) > 0){ + auto it = tablets_->find(tag); + CHECK(it != tablets_->end()) << "tag " << tag << " not exist."; + fs::SerializeToFile(it->second, tablet_path(dir, tag)); + } } + modified_tablet_set_.clear(); +} + +Storage* Storage::parent() { + return this; +} + +void Storage::MarkTabletModified(const std::string tag) { + modified_tablet_set_.insert(tag); } +void Storage::AddTag(const std::string& x) { + *data_->add_tags() = x; + WRITE_GUARD + } + +// StorageReader std::vector StorageReader::all_tags() { storage::Storage storage; Reload(storage); diff --git a/visualdl/storage/storage.h b/visualdl/storage/storage.h index 745bd0047..23d725f6e 100644 --- a/visualdl/storage/storage.h +++ b/visualdl/storage/storage.h @@ -67,8 +67,8 @@ struct Storage { Tablet AddTablet(const std::string& x); // Set storage's directory. - void SetDir(const std::string& dir) { *dir_ = dir; } - std::string dir() const { return *dir_; } + void SetDir(const std::string& dir); + std::string dir() const; // Save content in memory to `dir_`. void PersistToDisk(); @@ -77,20 +77,20 @@ struct Storage { void PersistToDisk(const std::string& dir); // A trick help to retrieve the storage's `SimpleSyncMeta`. - Storage* parent() { return this; } + Storage* parent(); + + void MarkTabletModified(const std::string tag); protected: // Add a tag which content is `x`. - void AddTag(const std::string& x) { - *data_->add_tags() = x; - WRITE_GUARD - } + void AddTag(const std::string& x); private: std::shared_ptr dir_; std::shared_ptr> tablets_; std::shared_ptr data_; std::shared_ptr> modes_; + std::set modified_tablet_set_; }; // Storage reader, each method will trigger a reading from disk. diff --git a/visualdl/storage/tablet.cc b/visualdl/storage/tablet.cc index bdab72f1c..fc8db3783 100644 --- a/visualdl/storage/tablet.cc +++ b/visualdl/storage/tablet.cc @@ -13,9 +13,25 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "visualdl/storage/tablet.h" +#include "visualdl/storage/storage.h" namespace visualdl { +void Tablet::SetTag(const std::string& mode, const std::string& tag) { + auto internal_tag = mode + "/" + tag; + string::TagEncode(internal_tag); + internal_encoded_tag_ = internal_tag; + data_->set_tag(internal_tag); + WRITE_GUARD +} + +Record Tablet::AddRecord() { + parent()->MarkTabletModified(internal_encoded_tag_); + IncTotalRecords(); + WRITE_GUARD + return Record(data_->add_records(), parent()); +} + TabletReader Tablet::reader() { return TabletReader(*data_); } } // namespace visualdl diff --git a/visualdl/storage/tablet.h b/visualdl/storage/tablet.h index 03bb1fe67..d74a3c00e 100644 --- a/visualdl/storage/tablet.h +++ b/visualdl/storage/tablet.h @@ -29,11 +29,11 @@ struct TabletReader; * Tablet is a helper for operations on storage::Tablet. */ struct Tablet { - enum Type { kScalar = 0, kHistogram = 1, kImage = 2 }; + enum Type { kScalar = 0, kHistogram = 1, kImage = 2, kUnknown = 100}; DECL_GUARD(Tablet); - Tablet(storage::Tablet* x, Storage* parent) : data_(x), x_(parent) {} + Tablet(storage::Tablet* x, Storage* parent) : data_(x), x_(parent), internal_encoded_tag_("") {} static Type type(const std::string& name) { if (name == "scalar") { @@ -46,6 +46,7 @@ struct Tablet { return kImage; } LOG(ERROR) << "unknown component: " << name; + return kUnknown; } // write operations. @@ -59,18 +60,8 @@ struct Tablet { WRITE_GUARD } - void SetTag(const std::string& mode, const std::string& tag) { - auto internal_tag = mode + "/" + tag; - string::TagEncode(internal_tag); - data_->set_tag(internal_tag); - WRITE_GUARD - } - - Record AddRecord() { - IncTotalRecords(); - WRITE_GUARD - return Record(data_->add_records(), parent()); - } + void SetTag(const std::string& mode, const std::string& tag); + Record AddRecord(); template Entry MutableMeta() { @@ -102,6 +93,7 @@ struct Tablet { private: Storage* x_; storage::Tablet* data_{nullptr}; + std::string internal_encoded_tag_; }; /* From 87f0fb329e6e44d75f3e6a7330fc5072652f1875 Mon Sep 17 00:00:00 2001 From: "Wang,Jeff" Date: Fri, 9 Mar 2018 11:47:09 -0800 Subject: [PATCH 2/3] Persist the data in the destructor Use unordered_set first. --- visualdl/storage/storage.cc | 6 +++++- visualdl/storage/storage.h | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/visualdl/storage/storage.cc b/visualdl/storage/storage.cc index 5f4b3e7b8..29a8704a7 100644 --- a/visualdl/storage/storage.cc +++ b/visualdl/storage/storage.cc @@ -21,7 +21,7 @@ Storage::Storage() { data_ = std::make_shared(); tablets_ = std::make_shared>(); modes_ = std::make_shared>(); - modified_tablet_set_ = std::set(); + modified_tablet_set_ = std::unordered_set(); time_t t; time(&t); data_->set_timestamp(t); @@ -32,6 +32,10 @@ Storage::Storage(const Storage& other) dir_ = other.dir_; } +Storage::~Storage() { + PersistToDisk(); +} + void Storage::AddMode(const std::string& x) { // avoid duplicate modes. if (modes_->count(x) != 0) return; diff --git a/visualdl/storage/storage.h b/visualdl/storage/storage.h index 23d725f6e..2b4fc099a 100644 --- a/visualdl/storage/storage.h +++ b/visualdl/storage/storage.h @@ -58,7 +58,7 @@ struct Storage { Storage(); Storage(const Storage& other); - + ~Storage(); // Add a mode. Mode is similar to TB's FileWriter, It can be "train" or "test" // or something else. void AddMode(const std::string& x); @@ -90,7 +90,7 @@ struct Storage { std::shared_ptr> tablets_; std::shared_ptr data_; std::shared_ptr> modes_; - std::set modified_tablet_set_; + std::unordered_set modified_tablet_set_; }; // Storage reader, each method will trigger a reading from disk. From 0b6a6b0067cf21a4139ac75c3b2b9c3657967e7d Mon Sep 17 00:00:00 2001 From: "Wang,Jeff" Date: Fri, 9 Mar 2018 11:51:26 -0800 Subject: [PATCH 3/3] Use -1 for the unknown. --- visualdl/storage/tablet.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/visualdl/storage/tablet.h b/visualdl/storage/tablet.h index d74a3c00e..b0907445b 100644 --- a/visualdl/storage/tablet.h +++ b/visualdl/storage/tablet.h @@ -29,7 +29,7 @@ struct TabletReader; * Tablet is a helper for operations on storage::Tablet. */ struct Tablet { - enum Type { kScalar = 0, kHistogram = 1, kImage = 2, kUnknown = 100}; + enum Type { kScalar = 0, kHistogram = 1, kImage = 2, kUnknown = -1}; DECL_GUARD(Tablet);