test examples by distrioptimizerv2 (#3007)

* enable scala examples by distrioptimizerv2 * update example's readme * update integration test
intel-analytics · Jun 30, 2020 · a8687a3 · a8687a3
1 parent 0316f09
commit a8687a3
Show file tree

Hide file tree

Showing 25 changed files with 144 additions and 35 deletions.
diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/languagemodel/PTBWordLM.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/languagemodel/PTBWordLM.scala
@@ -22,7 +22,7 @@ import com.intel.analytics.bigdl.dataset.{DataSet, SampleToMiniBatch}
 import com.intel.analytics.bigdl.nn.{CrossEntropyCriterion, Module, TimeDistributedCriterion}
 import com.intel.analytics.bigdl.optim._
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric._
-import com.intel.analytics.bigdl.utils.Engine
+import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2}
 import org.apache.log4j.{Level, Logger}
 import org.apache.spark.SparkContext
 import com.intel.analytics.bigdl.example.languagemodel.Utils._
@@ -82,6 +82,13 @@ object PTBWordLM {
           keepProb = param.keepProb)
       }
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         OptimMethod.load[Float](param.stateSnapshot.get)
       } else {

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/languagemodel/README.md b/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/languagemodel/README.md
@@ -47,4 +47,5 @@ In the above commands:
 ```--numSteps```: number of words per record in LM
 ```--overWrite```: do overwrite when saving checkpoint
 ```--keepProb```: the probability to do dropout
-```--withTransformerModel```: use transformer model in this LM
+```--withTransformerModel```: use transformer model in this LM
+```--optimizerVersion```: option can be used to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2"
diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/languagemodel/Utils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/languagemodel/Utils.scala
@@ -51,7 +51,8 @@ object Utils {
                           numSteps: Int = 20,
                           overWriteCheckpoint: Boolean = false,
                           keepProb: Float = 2.0f,
-                          withTransformerModel: Boolean = false)
+                          withTransformerModel: Boolean = false,
+                          optimizerVersion: Option[String] = None)
 
   val trainParser = new OptionParser[TrainParams]("BigDL ptbModel Train Example") {
     opt[String]('f', "dataFolder")
@@ -114,5 +115,9 @@ object Utils {
     opt[Boolean]("withTransformerModel")
       .text("Use transformer model in this LM")
       .action((x, c) => c.copy(withTransformerModel = true))
+
+    opt[String]("optimizerVersion")
+      .text("state optimizer version")
+      .action((x, c) => c.copy(optimizerVersion = Some(x)))
   }
 }
diff --git a/...dl/src/main/scala/com/intel/analytics/bigdl/example/treeLSTMSentiment/README.md b/...dl/src/main/scala/com/intel/analytics/bigdl/example/treeLSTMSentiment/README.md
@@ -65,5 +65,6 @@ Next just run the following command to run the code:
              --regRate           # number of L2 regularization rate, default is 1e-4
              --p                 # number of dropout probability rate, default is 0.5
              --epoch             # number of epochs, default is 5
+             --optimizerVersion  # option to set DistriOptimizer version, default is "optimizerV1"
     ```
 
diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/treeLSTMSentiment/Train.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/treeLSTMSentiment/Train.scala
@@ -23,7 +23,7 @@ import com.intel.analytics.bigdl.nn.{TimeDistributedCriterion, _}
 import com.intel.analytics.bigdl.numeric.NumericFloat
 import com.intel.analytics.bigdl.optim._
 import com.intel.analytics.bigdl.tensor.Tensor
-import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T}
+import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, OptimizerV1, OptimizerV2, T}
 import org.apache.log4j.{Level => Levle4j, Logger => Logger4j}
 import org.apache.spark.SparkContext
 import org.slf4j.{Logger, LoggerFactory}
@@ -93,6 +93,13 @@ object Train {
     val trainRDD = toSample(trainTreeRDD, trainLabelRDD, trainSentenceRDD)
     val devRDD = toSample(devTreeRDD, devLabelRDD, devSentenceRDD)
 
+    if (param.optimizerVersion.isDefined) {
+      param.optimizerVersion.get.toLowerCase match {
+        case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+        case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+      }
+    }
+
     val optimizer = Optimizer(
       model = TreeLSTMSentiment(word2VecTensor, param.hiddenSize, classNum, param.p),
       sampleRDD = trainRDD,

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/treeLSTMSentiment/Utils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/example/treeLSTMSentiment/Utils.scala
@@ -234,6 +234,9 @@ object Utils {
     opt[String]('e', "epoch")
       .text("max epoch")
       .action((x, c) => c.copy(epoch = x.toInt))
+    opt[String]("optimizerVersion")
+      .text("state optimizer version")
+      .action((x, c) => c.copy(optimizerVersion = Some(x)))
   }
 
   case class TreeLSTMSentimentParam (
@@ -243,6 +246,7 @@ object Utils {
     override val learningRate: Double = 0.05,
     regRate: Double = 1e-4,
     p: Double = 0.5,
-    epoch: Int = 5
+    epoch: Int = 5,
+    optimizerVersion: Option[String] = None
   ) extends AbstractTextClassificationParams
 }
diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/autoencoder/Train.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/autoencoder/Train.scala
@@ -26,7 +26,7 @@ import com.intel.analytics.bigdl.optim._
 import com.intel.analytics.bigdl.tensor.Tensor
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric._
-import com.intel.analytics.bigdl.utils.{Engine, T, Table}
+import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2, T, Table}
 import org.apache.log4j.{Level, Logger}
 import org.apache.spark.SparkContext
 
@@ -73,6 +73,13 @@ object Train {
         if (param.graphModel) Autoencoder.graph(classNum = 32) else Autoencoder(classNum = 32)
       }
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         OptimMethod.load[Float](param.stateSnapshot.get)
       } else {

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/autoencoder/Utils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/autoencoder/Utils.scala
@@ -34,7 +34,8 @@ object Utils {
     stateSnapshot: Option[String] = None,
     batchSize: Int = 150,
     maxEpoch: Int = 10,
-    graphModel: Boolean = false
+    graphModel: Boolean = false,
+    optimizerVersion: Option[String] = None
   )
 
   val trainParser = new OptionParser[TrainParams]("BigDL Autoencoder on MNIST") {
@@ -59,6 +60,9 @@ object Utils {
     opt[Unit]('g', "graphModel")
       .text("use graph model")
       .action((x, c) => c.copy(graphModel = true))
+    opt[String]("optimizerVersion")
+      .text("state optimizer version")
+      .action((x, c) => c.copy(optimizerVersion = Some(x)))
   }
 
   private[bigdl] def load(featureFile: Path, labelFile: Path): Array[ByteRecord] = {

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/inception/Options.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/inception/Options.scala
@@ -39,7 +39,8 @@ object Options {
     warmupEpoch: Option[Int] = None,
     gradientL2NormThreshold: Option[Double] = None,
     gradientMin: Option[Double] = None,
-    gradientMax: Option[Double] = None
+    gradientMax: Option[Double] = None,
+    optimizerVersion: Option[String] = None
   )
 
   val trainParser = new OptionParser[TrainParams]("BigDL Inception Example") {
@@ -99,6 +100,9 @@ object Options {
     opt[Double]("gradientMin")
       .text("min gradient clipping by")
       .action((x, c) => c.copy(gradientMin = Some(x)))
+    opt[String]("optimizerVersion")
+      .text("state optimizer version")
+      .action((x, c) => c.copy(optimizerVersion = Some(x)))
   }
 
   case class TestParams(

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/inception/README.md b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/inception/README.md
@@ -95,6 +95,7 @@ policy.
 * --gradientL2NormThreshold: optional. Gradient L2-Norm threshold used for norm2 gradient clipping.
 * --gradientMin: optional. Max gradient clipping by value, used in constant gradient clipping.
 * --gradientMax: optional. Min gradient clipping by value, used in constant gradient clipping.
+* --optimizerVersion: option can be used to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2".
 
 ## Test the Model
 * Spark standalone, example command

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/inception/Train.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/inception/Train.scala
@@ -19,7 +19,7 @@ import com.intel.analytics.bigdl._
 import com.intel.analytics.bigdl.nn.{ClassNLLCriterion, Module}
 import com.intel.analytics.bigdl.optim.SGD.{MultiStep, Poly, SequentialSchedule, Warmup}
 import com.intel.analytics.bigdl.optim._
-import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}
+import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, OptimizerV1, OptimizerV2, T, Table}
 import org.apache.log4j.{Level, Logger}
 import org.apache.spark.SparkContext
 
@@ -71,6 +71,13 @@ object TrainInceptionV1 {
 
       val warmupIteration = param.warmupEpoch.getOrElse(0) * iterationPerEpoch
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         OptimMethod.load[Float](param.stateSnapshot.get)
       } else {

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/lenet/README.md b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/lenet/README.md
@@ -111,3 +111,4 @@ In the above command
 * -f: where you put your MNIST data
 * --model: the model snapshot file
 * -b: The mini-batch size. It is expected that the mini-batch size is a multiple of node_number * core_number.
+* --optimizerVersion: option can be used to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2".
diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/lenet/Train.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/lenet/Train.scala
@@ -62,6 +62,13 @@ object Train {
         case MklDnn => CrossEntropyCriterion()
       }
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         OptimMethod.load[Float](param.stateSnapshot.get)
       } else {

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/lenet/Utils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/lenet/Utils.scala
@@ -20,7 +20,7 @@ import java.nio.ByteBuffer
 import java.nio.file.{Files, Path, Paths}
 
 import com.intel.analytics.bigdl.dataset.ByteRecord
-import com.intel.analytics.bigdl.utils.File
+import com.intel.analytics.bigdl.utils.{File, OptimizerVersion}
 import scopt.OptionParser
 
 object Utils {
@@ -42,7 +42,8 @@ object Utils {
     coreNumber: Int = -1,
     nodeNumber: Int = -1,
     overWriteCheckpoint: Boolean = false,
-    graphModel: Boolean = false
+    graphModel: Boolean = false,
+    optimizerVersion: Option[String] = None
   )
 
   val trainParser = new OptionParser[TrainParams]("BigDL Lenet Train Example") {
@@ -79,6 +80,9 @@ object Utils {
     opt[Unit]('g', "graphModel")
       .text("use graph model")
       .action((x, c) => c.copy(graphModel = true))
+    opt[String]("optimizerVersion")
+      .text("state optimizer version")
+      .action((x, c) => c.copy(optimizerVersion = Some(x)))
   }
 
   case class TestParams(

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/README.md b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/README.md
@@ -73,6 +73,7 @@ We support Local and Spark versions of training. Users can define <code>env</cod
     --weightDecay   [default 1e-4]
     --dampening     [default 0.0]
     --nesterov      [default true]
+    --optimizerVersion  [distriOptimizer version, default "optimizerV1"]
 ```
 ## Train ResNet on ImageNet
 This example shows the best practise we've experimented in multi-node training

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/TrainCIFAR10.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/TrainCIFAR10.scala
@@ -20,7 +20,7 @@ import com.intel.analytics.bigdl.nn.{CrossEntropyCriterion, Module}
 import com.intel.analytics.bigdl._
 import com.intel.analytics.bigdl.models.resnet.ResNet.{DatasetType, ShortcutType}
 import com.intel.analytics.bigdl.optim._
-import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}
+import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, OptimizerV1, OptimizerV2, T, Table}
 import org.apache.log4j.{Level, Logger}
 import org.apache.spark.SparkContext
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric._
@@ -73,6 +73,13 @@ object TrainCIFAR10 {
         curModel
       }
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         OptimMethod.load[Float](param.stateSnapshot.get)
       } else {

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/TrainImageNet.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/TrainImageNet.scala
@@ -89,6 +89,13 @@ object TrainImageNet {
 
       println(model)
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         val optim = OptimMethod.load[Float](param.stateSnapshot.get).asInstanceOf[SGD[Float]]
         val baseLr = param.learningRate

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/Utils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/resnet/Utils.scala
@@ -44,7 +44,8 @@ object Utils {
     nesterov: Boolean = true,
     graphModel: Boolean = false,
     warmupEpoch: Int = 0,
-    maxLr: Double = 0.0)
+    maxLr: Double = 0.0,
+    optimizerVersion: Option[String] = None)
 
   val trainParser = new OptionParser[TrainParams]("BigDL ResNet Example") {
     head("Train ResNet model on single node")
@@ -102,6 +103,9 @@ object Utils {
     opt[Double]("maxLr")
       .text("maxLr")
       .action((x, c) => c.copy(maxLr = x))
+    opt[String]("optimizerVersion")
+      .text("state optimizer version")
+      .action((x, c) => c.copy(optimizerVersion = Some(x)))
   }
 
   case class TestParams(

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/rnn/Train.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/rnn/Train.scala
@@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dataset.text.utils.SentenceToken
 import com.intel.analytics.bigdl.nn.{CrossEntropyCriterion, Module, TimeDistributedCriterion}
 import com.intel.analytics.bigdl.optim._
 import com.intel.analytics.bigdl.tensor.{Storage, Tensor}
-import com.intel.analytics.bigdl.utils.{Engine, T, Table}
+import com.intel.analytics.bigdl.utils.{Engine, OptimizerV1, OptimizerV2, T, Table}
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric._
 import org.apache.log4j.{Level, Logger}
 import org.apache.spark.SparkContext
@@ -101,6 +101,13 @@ object Train {
         curModel
       }
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         OptimMethod.load[Float](param.stateSnapshot.get)
       } else {

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/rnn/Utils.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/rnn/Utils.scala
@@ -51,7 +51,8 @@ object Utils {
                           nEpochs: Int = 30,
                           sentFile: Option[String] = None,
                           tokenFile: Option[String] = None,
-                          overWriteCheckpoint: Boolean = false)
+                          overWriteCheckpoint: Boolean = false,
+                          optimizerVersion: Option[String] = None)
 
   val trainParser = new OptionParser[TrainParams]("BigDL SimpleRNN Train Example") {
     opt[String]('f', "dataFolder")
@@ -123,6 +124,9 @@ object Utils {
     opt[Unit]("overWrite")
       .text("overwrite checkpoint files")
       .action( (_, c) => c.copy(overWriteCheckpoint = true) )
+    opt[String]("optimizerVersion")
+      .text("state optimizer version")
+      .action((x, c) => c.copy(optimizerVersion = Some(x)))
   }
 
   case class TestParams(

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/vgg/README.md b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/vgg/README.md
@@ -58,6 +58,7 @@ there are some files already exist in the folder, the old file will not be overw
 safety of your model files.
 * -b: The mini-batch size. It is expected that the mini-batch size is a multiple of node_number * core_number.
 * --summary: Where you store the training metainfo, which can be visualized in tensorboard
+* --optimizerVersion: option to set DistriOptimizer version, the value can be "optimizerV1" or "optimizerV2".
 ## Test Model
 Example command for running in Spark local mode
 ```

diff --git a/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/vgg/Train.scala b/spark/dl/src/main/scala/com/intel/analytics/bigdl/models/vgg/Train.scala
@@ -24,7 +24,7 @@ import com.intel.analytics.bigdl.dataset.image._
 import com.intel.analytics.bigdl.nn.{ClassNLLCriterion, Module}
 import com.intel.analytics.bigdl.optim._
 import com.intel.analytics.bigdl.tensor.TensorNumericMath.TensorNumeric._
-import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, T, Table}
+import com.intel.analytics.bigdl.utils.{Engine, LoggerFilter, OptimizerV1, OptimizerV2, T, Table}
 import com.intel.analytics.bigdl.visualization.{TrainSummary, ValidationSummary}
 import org.apache.log4j.{Level, Logger}
 import org.apache.spark.SparkContext
@@ -53,6 +53,13 @@ object Train {
         if (param.graphModel) VggForCifar10.graph(classNum = 10) else VggForCifar10(classNum = 10)
       }
 
+      if (param.optimizerVersion.isDefined) {
+        param.optimizerVersion.get.toLowerCase match {
+          case "optimizerv1" => Engine.setOptimizerVersion(OptimizerV1)
+          case "optimizerv2" => Engine.setOptimizerVersion(OptimizerV2)
+        }
+      }
+
       val optimMethod = if (param.stateSnapshot.isDefined) {
         OptimMethod.load[Float](param.stateSnapshot.get)
       } else {