diff --git a/visualdl/logic/sdk.h b/visualdl/logic/sdk.h index cabc08929..88b959904 100644 --- a/visualdl/logic/sdk.h +++ b/visualdl/logic/sdk.h @@ -10,6 +10,10 @@ namespace visualdl { const static std::string kDefaultMode{"default"}; +/** + * LogWriter is common Data Structure used to write data + * into a low level storage data structure. + */ class LogWriter { public: LogWriter(const std::string& dir, int sync_cycle) { @@ -34,6 +38,9 @@ class LogWriter { return writer; } + /** + * create a new tablet + */ Tablet AddTablet(const std::string& tag) { // TODO(ChunweiYan) add string check here. auto tmp = mode_ + "/" + tag; @@ -51,6 +58,10 @@ class LogWriter { std::string mode_{kDefaultMode}; }; +/** + * LogReader is common Data Structure used to read data + * from a low level storage data structure. + */ class LogReader { public: LogReader(const std::string& dir) : reader_(dir) {} @@ -180,19 +191,21 @@ struct Image { writer_.SetNumSamples(num_samples); SetCaption(tablet.reader().tag()); } + void SetCaption(const std::string& c) { writer_.SetCaptions(std::vector({c})); } + /* - * Start a sample period. + * Start a sampling period. */ void StartSampling(); /* - * Will this sample will be taken. + * Will this sample be taken. */ int IsSampleTaken(); /* - * End a sample period. + * End a sampling period. */ void FinishSampling(); @@ -265,6 +278,9 @@ struct ImageReader { std::string mode_; }; +/* + * Histogram component writer. + */ template struct Histogram { Histogram(Tablet tablet, int num_buckets) @@ -279,6 +295,9 @@ struct Histogram { Tablet writer_; }; +/* + * Histogram reader. + */ template struct HistogramReader { HistogramReader(TabletReader tablet) : reader_(tablet) {} diff --git a/visualdl/python/storage.py b/visualdl/python/storage.py index 311a7bf14..fd3aa2f76 100644 --- a/visualdl/python/storage.py +++ b/visualdl/python/storage.py @@ -6,28 +6,60 @@ class LogReader(object): - - cur_mode = None + """LogReader is a Python wrapper to read and analysis the data that + saved with data format defined in storage.proto. user can get + Scalar Reader/Image Reader/Histogram Reader from this module and use + them to reade the data you need. + """ def __init__(self, dir, reader=None): + """ + create a LogReader + :param dir: the dir where log file is. + :param reader: create a new LogReader with a formal one + """ self.dir = dir self.reader = reader if reader else core.LogReader(dir) def mode(self, mode): + """ + Set the current mode of reader. + + :param mode: the mode is something like a scope, it's used to + put some related data together. for example: train or test. + data generated during training can be marked mode train, and data + generated during testing can be marked test. + :return: the reader itself + """ self.reader.set_mode(mode) return self def as_mode(self, mode): + """ + create a new LogReader with mode and return it to user. + """ tmp = LogReader(dir, self.reader.as_mode(mode)) return tmp def modes(self): + """ + Get all modes of the log file + :return: + """ return self.reader.modes() - def tags(self, kind): - return self.reader.tags(kind) + def tags(self, component): + """ + Get all tags from the current log file for one kind of component + :param component: Scalar|Histogram|Images + :return: all the tags + """ + return self.reader.tags(component) def scalar(self, tag, type='float'): + """ + Get a scalar reader with tag and data type + """ type2scalar = { 'float': self.reader.get_scalar_float, 'double': self.reader.get_scalar_double, @@ -36,9 +68,15 @@ def scalar(self, tag, type='float'): return type2scalar[type](tag) def image(self, tag): + """ + Get a image reader with tag + """ return self.reader.get_image(tag) def histogram(self, tag, type='float'): + """ + Get a histogram reader with tag and data type + """ type2scalar = { 'float': self.reader.get_histogram_float, 'double': self.reader.get_histogram_double, @@ -54,6 +92,10 @@ def __exit__(self, type, value, traceback): class LogWriter(object): + """LogWriter is a Python wrapper to write data to log file with the data + format defined in storage.proto. user can get Scalar Reader/Image Reader/ + Histogram Reader from this module and use them to write the data to log file. + """ cur_mode = None @@ -67,13 +109,16 @@ def mode(self, mode): return self def as_mode(self, mode): + """ + create a new LogWriter with mode and return it. + """ LogWriter.cur_mode = LogWriter(self.dir, self.sync_cycle, self.writer.as_mode(mode)) return LogWriter.cur_mode def scalar(self, tag, type='float'): - ''' - Create a scalar component. - ''' + """ + Create a scalar writer with tag and type to write scalar data. + """ type2scalar = { 'float': self.writer.new_scalar_float, 'double': self.writer.new_scalar_double, @@ -82,15 +127,16 @@ def scalar(self, tag, type='float'): return type2scalar[type](tag) def image(self, tag, num_samples, step_cycle): - ''' - Create an image component. - ''' + """ + Create an image writer that used to write image data. + """ return self.writer.new_image(tag, num_samples, step_cycle) def histogram(self, tag, num_buckets, type='float'): - ''' - Create a histogram component. - ''' + """ + Create a histogram writer that used to write + histogram related data. + """ types = { 'float': self.writer.new_histogram_float, 'double': self.writer.new_histogram_double, diff --git a/visualdl/python/test_storage.py b/visualdl/python/test_storage.py index 7ca7c853e..5c78620a8 100644 --- a/visualdl/python/test_storage.py +++ b/visualdl/python/test_storage.py @@ -1,11 +1,10 @@ -import random -import time +import pprint +import sys import unittest import numpy as np from PIL import Image -import sys, pprint pprint.pprint(sys.path) from visualdl import LogWriter, LogReader @@ -119,7 +118,6 @@ def test_with_syntax(self): self.assertEqual(scalar.caption(), "train") def test_modes(self): - dir = "./tmp/storagetest0" store = LogWriter( self.dir, sync_cycle=1)