Skip to content

Commit

Permalink
NNFrames: support caching training data on Disk (intel#1588)
Browse files Browse the repository at this point in the history
* disk cache for NNFrames

* update doc and example

* update train method

* fix style

* fix ut

* fix ut
  • Loading branch information
YY-OnCall authored Sep 20, 2019
1 parent 015d7b1 commit 9420788
Showing 1 changed file with 23 additions and 0 deletions.
23 changes: 23 additions & 0 deletions python/dllib/test/bigdl/nnframes/test_nn_classifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,29 @@ def test_nnEstimator_fit_gradient_clipping(self):
estimator.setGradientClippingByL2Norm(1.2)
estimator.fit(df)

def test_nnEstimator_fit_with_Cache_Disk(self):
model = Sequential().add(Linear(2, 2))
criterion = MSECriterion()
estimator = NNEstimator(model, criterion, SeqToTensor([2]), ArrayToTensor([2])) \
.setBatchSize(1).setLearningRate(0.2).setMaxEpoch(2) \
.setDataCacheLevel("DISK_AND_DRAM", 2)

data = self.sc.parallelize([
((2.0, 1.0), (1.0, 2.0)),
((1.0, 2.0), (2.0, 1.0)),
((2.0, 1.0), (1.0, 2.0)),
((1.0, 2.0), (2.0, 1.0)),
((2.0, 1.0), (1.0, 2.0)),
((1.0, 2.0), (2.0, 1.0)),
((2.0, 1.0), (1.0, 2.0)),
((1.0, 2.0), (2.0, 1.0))])

schema = StructType([
StructField("features", ArrayType(DoubleType(), False), False),
StructField("label", ArrayType(DoubleType(), False), False)])
df = self.sqlContext.createDataFrame(data, schema)
estimator.fit(df)

def test_nnEstimator_fit_with_non_default_featureCol(self):
model = Sequential().add(Linear(2, 2))
criterion = MSECriterion()
Expand Down

0 comments on commit 9420788

Please sign in to comment.