From 5004f906502818118c13a1b229f4784871b7d068 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Fri, 12 Aug 2022 10:53:05 +0800
Subject: [PATCH 01/33] add a tensorflow example

---
 .../tensorflow_train_multi_instance.py        | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
new file mode 100644
index 00000000000..a628ec529e2
--- /dev/null
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -0,0 +1,111 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import tensorflow as tf
+from tensorflow.keras import layers
+from tensorflow.keras.applications import EfficientNetB0
+import tensorflow_datasets as tfds
+
+from bigdl.nano.tf.keras import Model, Sequential
+
+
+def create_datasets(img_size, batch_size):
+    (ds_train, ds_test), ds_info = tfds.load(
+        "stanford_dogs",
+        data_dir="../data/",
+        split=['train', 'test'],
+        with_info=True,
+        as_supervised=True
+    )
+
+    num_classes = ds_info.features['label'].num_classes
+
+    data_augmentation = Sequential([
+        layers.RandomFlip(),
+        layers.RandomRotation(factor=0.15),
+    ])
+
+    def preprocessing(img, label):
+        img, label =  tf.image.resize(img, (img_size, img_size)), tf.one_hot(label, num_classes)
+        return data_augmentation(img), label
+
+    AUTOTUNE = tf.data.AUTOTUNE
+    ds_train = ds_train.cache().repeat().map(preprocessing). \
+        batch(batch_size, drop_remainder=True).prefetch(AUTOTUNE)
+    ds_test = ds_test.map(preprocessing). \
+        batch(batch_size, drop_remainder=True).prefetch(AUTOTUNE)
+
+    return ds_train, ds_test, ds_info
+
+
+def create_model(num_classes, img_size, learning_rate=1e-2):
+    inputs = layers.Input(shape = (img_size, img_size, 3))
+
+    backbone = EfficientNetB0(include_top=False, input_tensor=inputs)
+
+    backbone.trainable = False
+
+    x = layers.GlobalAveragePooling2D(name='avg_pool')(backbone.output)
+    x = layers.BatchNormalization()(x)
+
+    top_dropout_rate = 0.2
+    x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
+    outputs = layers.Dense(num_classes, activation="softmax", name="pred")(x)
+
+    model = Model(inputs, outputs, name='EfficientNet')
+    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
+    model.compile(
+        loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
+    )
+    return model
+
+
+def unfreeze_model(model):
+    for layer in model.layers[-20:]:
+        if not isinstance(layer, layers.BatchNormalization):
+            layer.trainable = True
+    
+    optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)
+    model.compile(
+        loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
+    )
+
+
+if __name__ == '__main__':
+    img_size = 224
+    batch_size = 64
+    freezed_epochs = 15
+    unfreeze_epochs = 10
+    
+    ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
+    
+    num_classes = ds_info.features['label'].num_classes
+    steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
+    validation_steps = ds_info.splits['test'].num_examples // batch_size
+
+    model_default = create_model(num_classes=10, img_size=img_size)
+    
+    model_default.fit(ds_train,
+                      epochs=freezed_epochs,
+                      steps_per_epoch=steps_per_epoch,
+                      validation_data=ds_test,
+                      validation_steps=validation_steps)
+    unfreeze_model(model_default)
+    his_default = model_default.fit(ds_train,
+                                    epochs=unfreeze_epochs,
+                                    steps_per_epoch=steps_per_epoch,
+                                    validation_data=ds_test,
+                                    validation_steps=validation_steps)

From 22c11b8e3fc65caea4b9195cbeb1c21b5a22ed26 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 16:16:46 +0800
Subject: [PATCH 02/33] add tensorflow multi instance training

---
 .../tensorflow_train_multi_instance.py        | 48 ++++++++-----------
 1 file changed, 19 insertions(+), 29 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index a628ec529e2..fb49cebdfb3 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -15,17 +15,17 @@
 
 
 import tensorflow as tf
-from tensorflow.keras import layers
+from tensorflow.keras import layers, Sequential
 from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
-from bigdl.nano.tf.keras import Model, Sequential
+from bigdl.nano.tf.keras import Model
 
 
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
         "stanford_dogs",
-        data_dir="../data/",
+        data_dir="/tmp/data",
         split=['train', 'test'],
         with_info=True,
         as_supervised=True
@@ -73,22 +73,10 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
     return model
 
 
-def unfreeze_model(model):
-    for layer in model.layers[-20:]:
-        if not isinstance(layer, layers.BatchNormalization):
-            layer.trainable = True
-    
-    optimizer = tf.keras.optimizers.Adam(learning_rate=1e-4)
-    model.compile(
-        loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
-    )
-
-
 if __name__ == '__main__':
     img_size = 224
     batch_size = 64
-    freezed_epochs = 15
-    unfreeze_epochs = 10
+    num_epochs = 15
     
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     
@@ -96,16 +84,18 @@ def unfreeze_model(model):
     steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
     validation_steps = ds_info.splits['test'].num_examples // batch_size
 
-    model_default = create_model(num_classes=10, img_size=img_size)
-    
-    model_default.fit(ds_train,
-                      epochs=freezed_epochs,
-                      steps_per_epoch=steps_per_epoch,
-                      validation_data=ds_test,
-                      validation_steps=validation_steps)
-    unfreeze_model(model_default)
-    his_default = model_default.fit(ds_train,
-                                    epochs=unfreeze_epochs,
-                                    steps_per_epoch=steps_per_epoch,
-                                    validation_data=ds_test,
-                                    validation_steps=validation_steps)
+    # Multi-Instance Training
+    # 
+    # BigDL-Nano makes it very easy to conduct multi-instance training correctly.
+    # 
+    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create model,
+    # then just set the `num_processes` and `backend` parameter in the `fit` method.
+    # BigDL-Nano will launch the specific number of processes to perform data-parallel training.
+    model = create_model(num_classes=num_classes, img_size=img_size)
+    model.fit(ds_train,
+              epochs=num_epochs,
+              steps_per_epoch=steps_per_epoch,
+              validation_data=ds_test,
+              validation_steps=validation_steps,
+              num_processes=4,
+              backend='multiprocessing')

From 812f09ee4cff9b1e1eb55e47ea6ac1f75f41d016 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 16:29:00 +0800
Subject: [PATCH 03/33] add run script

---
 .github/workflows/nano_notebooks_tests.yml                 | 1 +
 .../training/tensorflow/run-nano-tensorflow-test.sh        | 7 +++++++
 .../training/tensorflow/tensorflow_train_multi_instance.py | 2 +-
 3 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index e01535974bc..5781eb5d9e3 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -67,6 +67,7 @@ jobs:
             bash python/nano/notebooks/pytorch/tutorial/run-nano-notebooks-pytorch-tutorial-tests.sh false
             bash python/nano/tutorial/inference/pytorch/run_nano_pytorch_inference_tests_onnx.sh
             bash python/nano/tutorial/training/pytorch-lightning/run_nano_pytorch_lightning_test.sh
+            bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
             source $CONDA/bin/deactivate
             $CONDA/bin/conda remove -n notebooks-pytorch --all
         env:
diff --git a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
new file mode 100644
index 00000000000..8b1331f5bf1
--- /dev/null
+++ b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
@@ -0,0 +1,7 @@
+export ANALYTICS_ZOO_ROOT=${ANALYTICS_ZOO_ROOT}
+export NANO_HOME=${ANALYTICS_ZOO_ROOT}/python/nano/src
+export NANO_TUTORIAL_TEST_DIR=${ANALYTICS_ZOO_ROOT}/python/nano/tutorial/training/tensorflow
+
+set -e
+
+python $NANO_TUTORIAL_TEST_DIR/tensorflow_train_multi_instance.py
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index fb49cebdfb3..ea38e0c2715 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -76,7 +76,7 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
 if __name__ == '__main__':
     img_size = 224
     batch_size = 64
-    num_epochs = 15
+    num_epochs = 1
     
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     

From 57d085f666d33cb88012acf54329e78990ed5bf1 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 16:35:31 +0800
Subject: [PATCH 04/33] add more description

---
 .../training/tensorflow/tensorflow_train_multi_instance.py    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index ea38e0c2715..0c57238c4c7 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -86,11 +86,15 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
 
     # Multi-Instance Training
     # 
+    # It is often beneficial to use multiple instances for training
+    # if a server contains multiple sockets or many cores, 
+    # so that the workload can make full use of all CPU cores.
     # BigDL-Nano makes it very easy to conduct multi-instance training correctly.
     # 
     # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create model,
     # then just set the `num_processes` and `backend` parameter in the `fit` method.
     # BigDL-Nano will launch the specific number of processes to perform data-parallel training.
+    #
     model = create_model(num_classes=num_classes, img_size=img_size)
     model.fit(ds_train,
               epochs=num_epochs,

From 3944c6e7ce25238b43329d0f490a32d3c176dc19 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 16:49:04 +0800
Subject: [PATCH 05/33] fix bugs

---
 .github/workflows/nano_notebooks_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index 5781eb5d9e3..eec427610ef 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -67,7 +67,6 @@ jobs:
             bash python/nano/notebooks/pytorch/tutorial/run-nano-notebooks-pytorch-tutorial-tests.sh false
             bash python/nano/tutorial/inference/pytorch/run_nano_pytorch_inference_tests_onnx.sh
             bash python/nano/tutorial/training/pytorch-lightning/run_nano_pytorch_lightning_test.sh
-            bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
             source $CONDA/bin/deactivate
             $CONDA/bin/conda remove -n notebooks-pytorch --all
         env:
@@ -145,6 +144,7 @@ jobs:
             pip install neural-compressor==1.11
             pip install tensorflow-datasets
             bash python/nano/notebooks/tensorflow/tutorial/run-nano-notebooks-tensorflow-tutorial-tests.sh
+            bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
             source $CONDA/bin/deactivate
             $CONDA/bin/conda remove -n notebooks-tutorial-tensorflow --all
         env:

From 9653102cad96a3b6d3afd2216c48c6a5b242cf86 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 16:57:41 +0800
Subject: [PATCH 06/33] update description

---
 .../training/tensorflow/tensorflow_train_multi_instance.py   | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 0c57238c4c7..07095c8ae27 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -19,8 +19,6 @@
 from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
-from bigdl.nano.tf.keras import Model
-
 
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
@@ -52,6 +50,9 @@ def preprocessing(img, label):
 
 
 def create_model(num_classes, img_size, learning_rate=1e-2):
+    # Use `Model` in `bigdl.nano.tf.keras` to replace tensorflow's `Model`
+    from bigdl.nano.tf.keras import Model
+
     inputs = layers.Input(shape = (img_size, img_size, 3))
 
     backbone = EfficientNetB0(include_top=False, input_tensor=inputs)

From 9a313def8ccc23d21a09b66400a8b8600026d8a1 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 18:39:41 +0800
Subject: [PATCH 07/33] fix bug

---
 .../training/tensorflow/tensorflow_train_multi_instance.py  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 07095c8ae27..de315ffb9fe 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -19,6 +19,9 @@
 from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
+# Use `Model` in `bigdl.nano.tf.keras` instead of tensorflow's `Model`
+from bigdl.nano.tf.keras import Model
+
 
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
@@ -50,9 +53,6 @@ def preprocessing(img, label):
 
 
 def create_model(num_classes, img_size, learning_rate=1e-2):
-    # Use `Model` in `bigdl.nano.tf.keras` to replace tensorflow's `Model`
-    from bigdl.nano.tf.keras import Model
-
     inputs = layers.Input(shape = (img_size, img_size, 3))
 
     backbone = EfficientNetB0(include_top=False, input_tensor=inputs)

From 060d7adde9a20183bdce40cc97b2362473d352ca Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 19:09:31 +0800
Subject: [PATCH 08/33] fix

---
 .../training/tensorflow/tensorflow_train_multi_instance.py      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index de315ffb9fe..cefda8e2d0f 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -76,7 +76,7 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
 
 if __name__ == '__main__':
     img_size = 224
-    batch_size = 64
+    batch_size = 32
     num_epochs = 1
     
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)

From 58493f06259f205513b6b90447b02d7494248618 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 19:30:46 +0800
Subject: [PATCH 09/33] fix

---
 .../training/tensorflow/tensorflow_train_multi_instance.py      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index cefda8e2d0f..32e6cd31f07 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -102,5 +102,5 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
               steps_per_epoch=steps_per_epoch,
               validation_data=ds_test,
               validation_steps=validation_steps,
-              num_processes=4,
+              num_processes=2,
               backend='multiprocessing')

From b805c38b75d11531b15316bb18406b3d2da055c2 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 19:54:33 +0800
Subject: [PATCH 10/33] fix

---
 .../training/tensorflow/tensorflow_train_multi_instance.py  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 32e6cd31f07..90d2bbcb684 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -15,12 +15,12 @@
 
 
 import tensorflow as tf
-from tensorflow.keras import layers, Sequential
+from tensorflow.keras import layers
 from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
-# Use `Model` in `bigdl.nano.tf.keras` instead of tensorflow's `Model`
-from bigdl.nano.tf.keras import Model
+# Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
+from bigdl.nano.tf.keras import Model, Sequential
 
 
 def create_datasets(img_size, batch_size):

From e5b699e6a73c1ed60001f1cd434c98a0ff01ac06 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 20:24:03 +0800
Subject: [PATCH 11/33] fix

---
 .../tensorflow/tensorflow_train_multi_instance.py      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 90d2bbcb684..afd458c4f38 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -15,19 +15,19 @@
 
 
 import tensorflow as tf
-from tensorflow.keras import layers
+from tensorflow.keras import layers, Sequential
 from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
 # Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
-from bigdl.nano.tf.keras import Model, Sequential
+from bigdl.nano.tf.keras import Model
 
 
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
         "stanford_dogs",
         data_dir="/tmp/data",
-        split=['train', 'test'],
+        split=['train[:10%]', 'test[:10%]'],
         with_info=True,
         as_supervised=True
     )
@@ -77,7 +77,7 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
 if __name__ == '__main__':
     img_size = 224
     batch_size = 32
-    num_epochs = 1
+    num_epochs = 10
     
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     
@@ -102,5 +102,5 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
               steps_per_epoch=steps_per_epoch,
               validation_data=ds_test,
               validation_steps=validation_steps,
-              num_processes=2,
+              num_processes=4,
               backend='multiprocessing')

From 0595b069566db774aa0419806d24d901a2436bc3 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 20:55:43 +0800
Subject: [PATCH 12/33] fix

---
 .../training/tensorflow/tensorflow_train_multi_instance.py    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index afd458c4f38..d1c12ad2fc5 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -82,8 +82,8 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     
     num_classes = ds_info.features['label'].num_classes
-    steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
-    validation_steps = ds_info.splits['test'].num_examples // batch_size
+    steps_per_epoch = ds_info.splits['train[:10%]'].num_examples // batch_size
+    validation_steps = ds_info.splits['test[:10%]'].num_examples // batch_size
 
     # Multi-Instance Training
     # 

From afdc371a91a27be5cd22ca4232b95d4e6ac819f3 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 21:22:32 +0800
Subject: [PATCH 13/33] fix

---
 .../training/tensorflow/tensorflow_train_multi_instance.py     | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index d1c12ad2fc5..39bcf612a34 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -26,8 +26,7 @@
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
         "stanford_dogs",
-        data_dir="/tmp/data",
-        split=['train[:10%]', 'test[:10%]'],
+        split=['train', 'test'],
         with_info=True,
         as_supervised=True
     )

From b5c9bd3c4260518ccca0ef18f3e33d999656093d Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 21:47:28 +0800
Subject: [PATCH 14/33] fix

---
 .github/workflows/nano_notebooks_tests.yml                  | 2 +-
 .../training/tensorflow/tensorflow_train_multi_instance.py  | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index eec427610ef..5c9eabc2a21 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -143,8 +143,8 @@ jobs:
             pip install jupyter nbconvert
             pip install neural-compressor==1.11
             pip install tensorflow-datasets
-            bash python/nano/notebooks/tensorflow/tutorial/run-nano-notebooks-tensorflow-tutorial-tests.sh
             bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+            bash python/nano/notebooks/tensorflow/tutorial/run-nano-notebooks-tensorflow-tutorial-tests.sh
             source $CONDA/bin/deactivate
             $CONDA/bin/conda remove -n notebooks-tutorial-tensorflow --all
         env:
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 39bcf612a34..d842f477db6 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -26,7 +26,7 @@
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
         "stanford_dogs",
-        split=['train', 'test'],
+        split=['train[:2048]', 'test[:2048]'],
         with_info=True,
         as_supervised=True
     )
@@ -81,8 +81,8 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     
     num_classes = ds_info.features['label'].num_classes
-    steps_per_epoch = ds_info.splits['train[:10%]'].num_examples // batch_size
-    validation_steps = ds_info.splits['test[:10%]'].num_examples // batch_size
+    steps_per_epoch = ds_info.splits['train[:2048]'].num_examples // batch_size
+    validation_steps = ds_info.splits['test[:2048]'].num_examples // batch_size
 
     # Multi-Instance Training
     # 

From a935106588bec946255e85ef2981e37a8329cc37 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 21:53:48 +0800
Subject: [PATCH 15/33] fix

---
 .../training/tensorflow/tensorflow_train_multi_instance.py  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index d842f477db6..d1a8ac29242 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -26,7 +26,7 @@
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
         "stanford_dogs",
-        split=['train[:2048]', 'test[:2048]'],
+        split=['train', 'test'],
         with_info=True,
         as_supervised=True
     )
@@ -81,8 +81,8 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     
     num_classes = ds_info.features['label'].num_classes
-    steps_per_epoch = ds_info.splits['train[:2048]'].num_examples // batch_size
-    validation_steps = ds_info.splits['test[:2048]'].num_examples // batch_size
+    steps_per_epoch = ds_info.splits['train[:5%]'].num_examples // batch_size
+    validation_steps = ds_info.splits['test[:5%]'].num_examples // batch_size
 
     # Multi-Instance Training
     # 

From 5564c7f192ed406adf07d695d623f16ac0b55c8b Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 23:19:17 +0800
Subject: [PATCH 16/33] no need to specify backend

---
 .github/workflows/nano_notebooks_tests.yml         | 14 +++++++++++++-
 .../tensorflow/tensorflow_train_multi_instance.py  | 11 +++++------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index 5c9eabc2a21..d22cc957c01 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -131,6 +131,19 @@ jobs:
           python -m pip install --upgrade setuptools==58.0.4
           python -m pip install --upgrade wheel
 
+      - name: Run TensorFlow Example
+        run: |
+          $CONDA/bin/conda create -n example-tensorflow -y python==3.7.10 setuptools=58.0.4
+          source $CONDA/bin/activate example-tensorflow
+          bash python/nano/dev/build_and_install.sh linux default false tensorflow
+          pip install tensorflow-datasets
+          source bigdl-nano-init
+          bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+          source $CONDA/bin/deactivate
+          $CONDA/bin/conda remove -n example-tensorflow --all
+        env:
+            ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
+
       - name: Run tutorial notebooks TensorFlow unit tests
         shell: bash
         run: |
@@ -143,7 +156,6 @@ jobs:
             pip install jupyter nbconvert
             pip install neural-compressor==1.11
             pip install tensorflow-datasets
-            bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
             bash python/nano/notebooks/tensorflow/tutorial/run-nano-notebooks-tensorflow-tutorial-tests.sh
             source $CONDA/bin/deactivate
             $CONDA/bin/conda remove -n notebooks-tutorial-tensorflow --all
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index d1a8ac29242..3a3859a2506 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -76,13 +76,13 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
 if __name__ == '__main__':
     img_size = 224
     batch_size = 32
-    num_epochs = 10
+    num_epochs = 1
     
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     
     num_classes = ds_info.features['label'].num_classes
-    steps_per_epoch = ds_info.splits['train[:5%]'].num_examples // batch_size
-    validation_steps = ds_info.splits['test[:5%]'].num_examples // batch_size
+    steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
+    validation_steps = ds_info.splits['test'].num_examples // batch_size
 
     # Multi-Instance Training
     # 
@@ -92,7 +92,7 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
     # BigDL-Nano makes it very easy to conduct multi-instance training correctly.
     # 
     # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create model,
-    # then just set the `num_processes` and `backend` parameter in the `fit` method.
+    # then just set the `num_processes` parameter in the `fit` method.
     # BigDL-Nano will launch the specific number of processes to perform data-parallel training.
     #
     model = create_model(num_classes=num_classes, img_size=img_size)
@@ -101,5 +101,4 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
               steps_per_epoch=steps_per_epoch,
               validation_data=ds_test,
               validation_steps=validation_steps,
-              num_processes=4,
-              backend='multiprocessing')
+              num_processes=4)

From a0873ee0128a875196bc76572ca1b14830f55f90 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Thu, 18 Aug 2022 23:25:19 +0800
Subject: [PATCH 17/33] fix issue

---
 .github/workflows/nano_notebooks_tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index d22cc957c01..a0e1587a022 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -136,6 +136,7 @@ jobs:
           $CONDA/bin/conda create -n example-tensorflow -y python==3.7.10 setuptools=58.0.4
           source $CONDA/bin/activate example-tensorflow
           bash python/nano/dev/build_and_install.sh linux default false tensorflow
+          pip install neural-compressor==1.11
           pip install tensorflow-datasets
           source bigdl-nano-init
           bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh

From 05fa997058d19271891bb19f9e4e3694da8614ef Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Fri, 19 Aug 2022 17:29:46 +0800
Subject: [PATCH 18/33] add tensorflow quantization example

---
 .github/workflows/nano_notebooks_tests.yml    |  1 +
 .../run-nano-tensorflow-inference-test.sh     |  7 ++
 .../tensorflow/tensorflow_quantization.py     | 91 +++++++++++++++++++
 3 files changed, 99 insertions(+)
 create mode 100644 python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
 create mode 100644 python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index a0e1587a022..ad96c8963ed 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -139,6 +139,7 @@ jobs:
           pip install neural-compressor==1.11
           pip install tensorflow-datasets
           source bigdl-nano-init
+          bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
           bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
           source $CONDA/bin/deactivate
           $CONDA/bin/conda remove -n example-tensorflow --all
diff --git a/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh b/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
new file mode 100644
index 00000000000..89178713040
--- /dev/null
+++ b/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
@@ -0,0 +1,7 @@
+export ANALYTICS_ZOO_ROOT=${ANALYTICS_ZOO_ROOT}
+export NANO_HOME=${ANALYTICS_ZOO_ROOT}/python/nano/src
+export NANO_TUTORIAL_TEST_DIR=${ANALYTICS_ZOO_ROOT}/python/nano/tutorial/inference/tensorflow
+
+set -e
+
+python $NANO_TUTORIAL_TEST_DIR/tensorflow_quantization.py
diff --git a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
new file mode 100644
index 00000000000..7916d90ff21
--- /dev/null
+++ b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
@@ -0,0 +1,91 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Required Dependecies
+
+# ```bash
+# pip install neural-compressor==1.11.0
+# ```
+
+
+import os
+import tensorflow as tf
+from tensorflow.keras import layers
+from tensorflow.keras.applications import ResNet50
+from tensorflow.keras.metrics import CategoricalAccuracy
+import tensorflow_datasets as tfds
+
+# Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
+from bigdl.nano.tf.keras import Model, Sequential
+
+
+def create_datasets(img_size, batch_size):
+    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
+                                          data_dir='/tmp/data',
+                                          split=['train', 'validation'],
+                                          with_info=True,
+                                          as_supervised=True)
+
+    # Create a Dataset that includes only 1/num_shards of full dataset.
+    num_shards = int(os.environ.get('NUM_SHARDS', 1))
+    train_ds = train_ds.shard(num_shards, index=0)
+    test_ds = test_ds.shard(num_shards, index=0)
+    num_classes = info.features['label'].num_classes
+    
+    train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+                                                tf.one_hot(label, num_classes))).batch(batch_size)
+    test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+                                              tf.one_hot(label, num_classes))).batch(batch_size)
+    return train_ds, test_ds, info
+
+
+def create_model(num_classes, img_size):
+    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
+    x = tf.cast(inputs, tf.float32)
+    x = tf.keras.applications.resnet50.preprocess_input(x)
+    backbone = ResNet50(weights='imagenet')
+    backbone.trainable = False
+    x = backbone(x)
+    x = layers.Dense(512, activation='relu')(x)
+    outputs = layers.Dense(num_classes, activation='softmax')(x)
+
+    model = Model(inputs=inputs, outputs=outputs)
+    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
+    return model
+
+
+if __name__ == '__main__':
+    img_size = 224
+    batch_size = 32
+
+    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
+    num_classes = ds_info.features['label'].num_classes
+
+    # Quantization with Intel Neural Compressor
+    #
+    # A quantized model executes tensor operations with reduced precision.
+    # This allows for a more compact model representation and the use of high 
+    # performance vectorized operations on many hardware platforms.
+    # It can be used to speed up inference and only the forward pass is supported
+    # for quantized operators.
+    # 
+    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
+    # `Model.quantize()` return a Keras module with desired precision and accuracy.
+    # Taking Resnet50 as an example, you can add quantization as below.
+    #
+    model = create_model(num_classes, img_size)
+    q_model = model.quantize(calib_dataset=test_ds,
+                             metric=CategoricalAccuracy(),
+                             tuning_strategy='basic')

From 48fd076b890274a792676d36b84e76b877618529 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Mon, 22 Aug 2022 17:02:17 +0800
Subject: [PATCH 19/33] add tensorflow sparse embedding example

---
 .../tensorflow/tensorflow_quantization.py     |   2 +
 .../tensorflow/run-nano-tensorflow-test.sh    |   2 +
 .../tensorflow/tensorflow_sparse_embedding.py | 121 ++++++++++++++++++
 .../tensorflow_train_multi_instance.py        |   6 +-
 4 files changed, 130 insertions(+), 1 deletion(-)
 create mode 100644 python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py

diff --git a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
index 7916d90ff21..80d100f8aca 100644
--- a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
+++ b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This example shows how to do quantization with bigdl-nano
+
 # Required Dependecies
 
 # ```bash
diff --git a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
index 8b1331f5bf1..4e5d5b62980 100644
--- a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+++ b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
@@ -4,4 +4,6 @@ export NANO_TUTORIAL_TEST_DIR=${ANALYTICS_ZOO_ROOT}/python/nano/tutorial/trainin
 
 set -e
 
+export num_epochs=1
+python $NANO_TUTORIAL_TEST_DIR/tensorflow_sparse_embedding.py
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_train_multi_instance.py
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py b/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
new file mode 100644
index 00000000000..c28c646b24a
--- /dev/null
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
@@ -0,0 +1,121 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This example shows how to use sparse adam optimizer and embedding layer with bigdl-nano
+
+
+import os
+import re
+import string
+
+import tensorflow as tf
+import tensorflow_datasets as tfds
+from tensorflow.keras import layers
+from tensorflow.keras.layers import TextVectorization
+
+# import `Embedding`, `SparseAdam` and `Model` from bigdl-nano
+from bigdl.nano.tf.keras.layers import Embedding
+from bigdl.nano.tf.optimizers import SparseAdam
+from bigdl.nano.tf.keras import Model
+
+
+max_features = 20000
+embedding_dim = 128
+
+
+def create_datasets():
+    (raw_train_ds, raw_val_ds, raw_test_ds), info = tfds.load(
+        "imdb_reviews",
+        data_dir="/tmp/data",
+        split=['train[:80%]', 'train[80%:]', 'test'],
+        as_supervised=True,
+        batch_size=32,
+        with_info=True
+    )
+
+    def custom_standardization(input_data):
+        lowercase = tf.strings.lower(input_data)
+        stripped_html = tf.strings.regex_replace(lowercase, "<br />", " ")
+        return tf.strings.regex_replace(
+            stripped_html, f"[{re.escape(string.punctuation)}]", ""
+        )
+
+    vectorize_layer = TextVectorization(
+        standardize=custom_standardization,
+        max_tokens=max_features,
+        output_mode="int",
+        output_sequence_length=500,
+    )
+
+    def vectorize_text(text, label):
+        text = tf.expand_dims(text, -1)
+        return vectorize_layer(text), label
+
+    # Vectorize the data and do prefetching / buffering.
+    train_ds = raw_train_ds.map(vectorize_text).cache().prefetch(buffer_size=10)
+    val_ds = raw_val_ds.map(vectorize_text).cache().prefetch(buffer_size=10)
+    test_ds = raw_test_ds.map(vectorize_text).cache().prefetch(buffer_size=10)
+
+    return train_ds, val_ds, test_ds
+
+
+def make_backbone():
+    inputs = tf.keras.Input(shape=(None, embedding_dim))
+    x = layers.Dropout(0.5)(inputs)
+    x = layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(x)
+    x = layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(x)
+    x = layers.GlobalMaxPooling1D()(x)
+    x = layers.Dense(128, activation="relu")(x)
+    x = layers.Dropout(0.5)(x)
+    predictions = layers.Dense(1, activation="sigmoid", name="predictions")(x)
+
+    model = Model(inputs, predictions)
+    return model
+
+
+def make_model():
+    inputs = tf.keras.Input(shape=(None,), dtype="int64")
+    
+    # use `Embedding` layer in `bigdl.nano.tf.keras.layers`
+    x = Embedding(max_features, embedding_dim)(inputs)
+
+    predictions = make_backbone()(x)
+    model = Model(inputs, predictions)
+
+    # use `SparseAdam` optimizer in `bigdl.nano.tf.optimizers`
+    model.compile(loss="binary_crossentropy", optimizer=SparseAdam(), metrics=["accuracy"])
+
+    return model
+
+
+if __name__=='__main__':
+    num_epochs = int(os.environ.get('num_epochs', 10))
+
+    train_ds, val_ds, test_ds = create_datasets()
+
+    # Use sparse adam optimizer and embedding layer
+    #
+    # Sparse embedding represents the gradient matrix by a sparse tensor and 
+    # only calculating gradients for embedding vectors which will be non zero.
+    # It can be used to speed up and reduce memory usage
+    # 
+    # Use `Embedding` in `bigdl.nano.tf.keras.layers` to create a sparse embedding layer,
+    # then use `SparseAdam` in `bigdl.nano.tf.optimizers` as the model's optimizer.
+    #
+    model = make_model()
+
+    model.fit(train_ds, validation_data=val_ds, epochs=num_epochs)
+
+    his = model.evaluate(test_ds)
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 3a3859a2506..7681d011761 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# This example shows how to do multi-process training with bigdl-nano
+
+
+import os
 
 import tensorflow as tf
 from tensorflow.keras import layers, Sequential
@@ -76,7 +80,7 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
 if __name__ == '__main__':
     img_size = 224
     batch_size = 32
-    num_epochs = 1
+    num_epochs = int(os.environ.get('num_epochs', 10))
     
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     

From 4c355b88c4cab62c1043ece035944b8f03de6591 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Mon, 22 Aug 2022 17:17:12 +0800
Subject: [PATCH 20/33] fix

---
 .../training/tensorflow/tensorflow_train_multi_instance.py       | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 7681d011761..2e32e06ede4 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -30,6 +30,7 @@
 def create_datasets(img_size, batch_size):
     (ds_train, ds_test), ds_info = tfds.load(
         "stanford_dogs",
+        data_dir="/tmp/data",
         split=['train', 'test'],
         with_info=True,
         as_supervised=True

From d3c3d6d0d78d4c6e18b64e183734ac0e4f5eb2f8 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Mon, 22 Aug 2022 17:28:55 +0800
Subject: [PATCH 21/33] fix

---
 .../inference/tensorflow/run-nano-tensorflow-inference-test.sh  | 1 +
 .../tutorial/training/tensorflow/run-nano-tensorflow-test.sh    | 2 +-
 .../tutorial/training/tensorflow/tensorflow_sparse_embedding.py | 2 +-
 .../training/tensorflow/tensorflow_train_multi_instance.py      | 2 +-
 4 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh b/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
index 89178713040..998fa6a4c77 100644
--- a/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
+++ b/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
@@ -4,4 +4,5 @@ export NANO_TUTORIAL_TEST_DIR=${ANALYTICS_ZOO_ROOT}/python/nano/tutorial/inferen
 
 set -e
 
+export NUM_SHARDS=4
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_quantization.py
diff --git a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
index 4e5d5b62980..5a212ba586e 100644
--- a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+++ b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
@@ -4,6 +4,6 @@ export NANO_TUTORIAL_TEST_DIR=${ANALYTICS_ZOO_ROOT}/python/nano/tutorial/trainin
 
 set -e
 
-export num_epochs=1
+export NUM_EPOCHS=1
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_sparse_embedding.py
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_train_multi_instance.py
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py b/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
index c28c646b24a..b2a3d5d4aea 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
@@ -101,7 +101,7 @@ def make_model():
 
 
 if __name__=='__main__':
-    num_epochs = int(os.environ.get('num_epochs', 10))
+    num_epochs = int(os.environ.get('NUM_EPOCHS', 10))
 
     train_ds, val_ds, test_ds = create_datasets()
 
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 2e32e06ede4..4cbea087c59 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -81,7 +81,7 @@ def create_model(num_classes, img_size, learning_rate=1e-2):
 if __name__ == '__main__':
     img_size = 224
     batch_size = 32
-    num_epochs = int(os.environ.get('num_epochs', 10))
+    num_epochs = int(os.environ.get('NUM_EPOCHS', 10))
     
     ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
     

From d85fadfde07591a4012a2cd1bb1d713c55e6c062 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Mon, 22 Aug 2022 18:14:29 +0800
Subject: [PATCH 22/33] ignore quantization example now

---
 .github/workflows/nano_notebooks_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index ad96c8963ed..7bf049476fd 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -139,7 +139,7 @@ jobs:
           pip install neural-compressor==1.11
           pip install tensorflow-datasets
           source bigdl-nano-init
-          bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
+          # bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
           bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
           source $CONDA/bin/deactivate
           $CONDA/bin/conda remove -n example-tensorflow --all

From 981a1b3c7578d9c2bd65b988dc2a1ea261d6e04b Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Mon, 22 Aug 2022 19:29:45 +0800
Subject: [PATCH 23/33] fix

---
 .../tensorflow/tensorflow_sparse_embedding.py         | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py b/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
index b2a3d5d4aea..9fa149e3186 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py
@@ -58,15 +58,18 @@ def custom_standardization(input_data):
         output_mode="int",
         output_sequence_length=500,
     )
+    
+    text_ds = raw_train_ds.map(lambda x, y: x)
+    vectorize_layer.adapt(text_ds)
 
     def vectorize_text(text, label):
         text = tf.expand_dims(text, -1)
         return vectorize_layer(text), label
 
-    # Vectorize the data and do prefetching / buffering.
-    train_ds = raw_train_ds.map(vectorize_text).cache().prefetch(buffer_size=10)
-    val_ds = raw_val_ds.map(vectorize_text).cache().prefetch(buffer_size=10)
-    test_ds = raw_test_ds.map(vectorize_text).cache().prefetch(buffer_size=10)
+    # Vectorize the data
+    train_ds = raw_train_ds.map(vectorize_text)
+    val_ds = raw_val_ds.map(vectorize_text)
+    test_ds = raw_test_ds.map(vectorize_text)
 
     return train_ds, val_ds, test_ds
 

From 4b0f5c2479933aedaf23b0cba32ace545e0e9003 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Mon, 22 Aug 2022 20:02:00 +0800
Subject: [PATCH 24/33] try to fix quantization

---
 .github/workflows/nano_notebooks_tests.yml    |   2 +-
 .../tensorflow/tensorflow_quantization.py     | 182 +++++++++++++-----
 2 files changed, 136 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index 7bf049476fd..ad96c8963ed 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -139,7 +139,7 @@ jobs:
           pip install neural-compressor==1.11
           pip install tensorflow-datasets
           source bigdl-nano-init
-          # bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
+          bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
           bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
           source $CONDA/bin/deactivate
           $CONDA/bin/conda remove -n example-tensorflow --all
diff --git a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
index 80d100f8aca..cdeea31f72b 100644
--- a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
+++ b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
@@ -22,72 +22,160 @@
 # ```
 
 
+# import os
+# import tensorflow as tf
+# from tensorflow.keras import layers
+# from tensorflow.keras.applications import ResNet50
+from tensorflow.keras.metrics import CategoricalAccuracy
+# import tensorflow_datasets as tfds
+
+# # Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
+# from bigdl.nano.tf.keras import Model, Sequential
+
+
+# def create_datasets(img_size, batch_size):
+#     (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
+#                                           data_dir='/tmp/data',
+#                                           split=['train', 'validation'],
+#                                           with_info=True,
+#                                           as_supervised=True)
+
+#     # Create a Dataset that includes only 1/num_shards of full dataset.
+#     num_shards = int(os.environ.get('NUM_SHARDS', 1))
+#     train_ds = train_ds.shard(num_shards, index=0)
+#     test_ds = test_ds.shard(num_shards, index=0)
+#     num_classes = info.features['label'].num_classes
+    
+#     train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+#                                                 tf.one_hot(label, num_classes))).batch(batch_size)
+#     test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+#                                               tf.one_hot(label, num_classes))).batch(batch_size)
+#     return train_ds, test_ds, info
+
+
+# def create_model(num_classes, img_size):
+#     inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
+#     x = tf.cast(inputs, tf.float32)
+#     x = tf.keras.applications.resnet50.preprocess_input(x)
+#     backbone = ResNet50(weights='imagenet')
+#     backbone.trainable = False
+#     x = backbone(x)
+#     x = layers.Dense(512, activation='relu')(x)
+#     outputs = layers.Dense(num_classes, activation='softmax')(x)
+
+#     model = Model(inputs=inputs, outputs=outputs)
+#     model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
+#     return model
+
+
+# if __name__ == '__main__':
+#     img_size = 224
+#     batch_size = 32
+
+#     train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
+#     num_classes = ds_info.features['label'].num_classes
+
+#     # Quantization with Intel Neural Compressor
+#     #
+#     # A quantized model executes tensor operations with reduced precision.
+#     # This allows for a more compact model representation and the use of high 
+#     # performance vectorized operations on many hardware platforms.
+#     # It can be used to speed up inference and only the forward pass is supported
+#     # for quantized operators.
+#     # 
+#     # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
+#     # `Model.quantize()` return a Keras module with desired precision and accuracy.
+#     # Taking Resnet50 as an example, you can add quantization as below.
+#     #
+#     model = create_model(num_classes, img_size)
+#     q_model = model.quantize(calib_dataset=test_ds,
+#                              metric=CategoricalAccuracy(),
+#                              tuning_strategy='basic')
+
 import os
+
 import tensorflow as tf
-from tensorflow.keras import layers
-from tensorflow.keras.applications import ResNet50
-from tensorflow.keras.metrics import CategoricalAccuracy
+from tensorflow.keras import layers, Sequential
+from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
 # Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
-from bigdl.nano.tf.keras import Model, Sequential
+from bigdl.nano.tf.keras import Model
 
 
 def create_datasets(img_size, batch_size):
-    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
-                                          data_dir='/tmp/data',
-                                          split=['train', 'validation'],
-                                          with_info=True,
-                                          as_supervised=True)
-
-    # Create a Dataset that includes only 1/num_shards of full dataset.
-    num_shards = int(os.environ.get('NUM_SHARDS', 1))
-    train_ds = train_ds.shard(num_shards, index=0)
-    test_ds = test_ds.shard(num_shards, index=0)
-    num_classes = info.features['label'].num_classes
-    
-    train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-                                                tf.one_hot(label, num_classes))).batch(batch_size)
-    test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-                                              tf.one_hot(label, num_classes))).batch(batch_size)
-    return train_ds, test_ds, info
-
-
-def create_model(num_classes, img_size):
-    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
-    x = tf.cast(inputs, tf.float32)
-    x = tf.keras.applications.resnet50.preprocess_input(x)
-    backbone = ResNet50(weights='imagenet')
+    (ds_train, ds_test), ds_info = tfds.load(
+        "stanford_dogs",
+        data_dir="/tmp/data",
+        split=['train', 'test'],
+        with_info=True,
+        as_supervised=True
+    )
+
+    num_classes = ds_info.features['label'].num_classes
+
+    data_augmentation = Sequential([
+        layers.RandomFlip(),
+        layers.RandomRotation(factor=0.15),
+    ])
+
+    def preprocessing(img, label):
+        img, label =  tf.image.resize(img, (img_size, img_size)), tf.one_hot(label, num_classes)
+        return data_augmentation(img), label
+
+    # AUTOTUNE = tf.data.AUTOTUNE
+    ds_train = ds_train.map(preprocessing).batch(batch_size, drop_remainder=True)
+    ds_test = ds_test.map(preprocessing).batch(batch_size, drop_remainder=True)
+
+    return ds_train, ds_test, ds_info
+
+
+def create_model(num_classes, img_size, learning_rate=1e-2):
+    inputs = layers.Input(shape = (img_size, img_size, 3))
+
+    backbone = EfficientNetB0(include_top=False, input_tensor=inputs)
+
     backbone.trainable = False
-    x = backbone(x)
-    x = layers.Dense(512, activation='relu')(x)
-    outputs = layers.Dense(num_classes, activation='softmax')(x)
 
-    model = Model(inputs=inputs, outputs=outputs)
-    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
+    x = layers.GlobalAveragePooling2D(name='avg_pool')(backbone.output)
+    x = layers.BatchNormalization()(x)
+
+    top_dropout_rate = 0.2
+    x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
+    outputs = layers.Dense(num_classes, activation="softmax", name="pred")(x)
+
+    model = Model(inputs, outputs, name='EfficientNet')
+    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
+    model.compile(
+        loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
+    )
     return model
 
 
 if __name__ == '__main__':
     img_size = 224
     batch_size = 32
-
-    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
+    num_epochs = int(os.environ.get('NUM_EPOCHS', 10))
+    
+    ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
+    
     num_classes = ds_info.features['label'].num_classes
+    steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
+    validation_steps = ds_info.splits['test'].num_examples // batch_size
 
-    # Quantization with Intel Neural Compressor
-    #
-    # A quantized model executes tensor operations with reduced precision.
-    # This allows for a more compact model representation and the use of high 
-    # performance vectorized operations on many hardware platforms.
-    # It can be used to speed up inference and only the forward pass is supported
-    # for quantized operators.
+    # Multi-Instance Training
+    # 
+    # It is often beneficial to use multiple instances for training
+    # if a server contains multiple sockets or many cores, 
+    # so that the workload can make full use of all CPU cores.
+    # BigDL-Nano makes it very easy to conduct multi-instance training correctly.
     # 
-    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
-    # `Model.quantize()` return a Keras module with desired precision and accuracy.
-    # Taking Resnet50 as an example, you can add quantization as below.
+    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create model,
+    # then just set the `num_processes` parameter in the `fit` method.
+    # BigDL-Nano will launch the specific number of processes to perform data-parallel training.
     #
-    model = create_model(num_classes, img_size)
-    q_model = model.quantize(calib_dataset=test_ds,
+    model = create_model(num_classes=num_classes, img_size=img_size)
+    q_model = model.quantize(calib_dataset=ds_test,
                              metric=CategoricalAccuracy(),
                              tuning_strategy='basic')
+

From 2152c195519fa1ecac324d4de396e0d80db57df0 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Mon, 22 Aug 2022 20:19:45 +0800
Subject: [PATCH 25/33] recover quantization example

---
 .../tensorflow/tensorflow_quantization.py     | 182 +++++-------------
 1 file changed, 47 insertions(+), 135 deletions(-)

diff --git a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
index cdeea31f72b..80d100f8aca 100644
--- a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
+++ b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
@@ -22,160 +22,72 @@
 # ```
 
 
-# import os
-# import tensorflow as tf
-# from tensorflow.keras import layers
-# from tensorflow.keras.applications import ResNet50
-from tensorflow.keras.metrics import CategoricalAccuracy
-# import tensorflow_datasets as tfds
-
-# # Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
-# from bigdl.nano.tf.keras import Model, Sequential
-
-
-# def create_datasets(img_size, batch_size):
-#     (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
-#                                           data_dir='/tmp/data',
-#                                           split=['train', 'validation'],
-#                                           with_info=True,
-#                                           as_supervised=True)
-
-#     # Create a Dataset that includes only 1/num_shards of full dataset.
-#     num_shards = int(os.environ.get('NUM_SHARDS', 1))
-#     train_ds = train_ds.shard(num_shards, index=0)
-#     test_ds = test_ds.shard(num_shards, index=0)
-#     num_classes = info.features['label'].num_classes
-    
-#     train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-#                                                 tf.one_hot(label, num_classes))).batch(batch_size)
-#     test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-#                                               tf.one_hot(label, num_classes))).batch(batch_size)
-#     return train_ds, test_ds, info
-
-
-# def create_model(num_classes, img_size):
-#     inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
-#     x = tf.cast(inputs, tf.float32)
-#     x = tf.keras.applications.resnet50.preprocess_input(x)
-#     backbone = ResNet50(weights='imagenet')
-#     backbone.trainable = False
-#     x = backbone(x)
-#     x = layers.Dense(512, activation='relu')(x)
-#     outputs = layers.Dense(num_classes, activation='softmax')(x)
-
-#     model = Model(inputs=inputs, outputs=outputs)
-#     model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
-#     return model
-
-
-# if __name__ == '__main__':
-#     img_size = 224
-#     batch_size = 32
-
-#     train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
-#     num_classes = ds_info.features['label'].num_classes
-
-#     # Quantization with Intel Neural Compressor
-#     #
-#     # A quantized model executes tensor operations with reduced precision.
-#     # This allows for a more compact model representation and the use of high 
-#     # performance vectorized operations on many hardware platforms.
-#     # It can be used to speed up inference and only the forward pass is supported
-#     # for quantized operators.
-#     # 
-#     # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
-#     # `Model.quantize()` return a Keras module with desired precision and accuracy.
-#     # Taking Resnet50 as an example, you can add quantization as below.
-#     #
-#     model = create_model(num_classes, img_size)
-#     q_model = model.quantize(calib_dataset=test_ds,
-#                              metric=CategoricalAccuracy(),
-#                              tuning_strategy='basic')
-
 import os
-
 import tensorflow as tf
-from tensorflow.keras import layers, Sequential
-from tensorflow.keras.applications import EfficientNetB0
+from tensorflow.keras import layers
+from tensorflow.keras.applications import ResNet50
+from tensorflow.keras.metrics import CategoricalAccuracy
 import tensorflow_datasets as tfds
 
 # Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
-from bigdl.nano.tf.keras import Model
+from bigdl.nano.tf.keras import Model, Sequential
 
 
 def create_datasets(img_size, batch_size):
-    (ds_train, ds_test), ds_info = tfds.load(
-        "stanford_dogs",
-        data_dir="/tmp/data",
-        split=['train', 'test'],
-        with_info=True,
-        as_supervised=True
-    )
-
-    num_classes = ds_info.features['label'].num_classes
-
-    data_augmentation = Sequential([
-        layers.RandomFlip(),
-        layers.RandomRotation(factor=0.15),
-    ])
-
-    def preprocessing(img, label):
-        img, label =  tf.image.resize(img, (img_size, img_size)), tf.one_hot(label, num_classes)
-        return data_augmentation(img), label
-
-    # AUTOTUNE = tf.data.AUTOTUNE
-    ds_train = ds_train.map(preprocessing).batch(batch_size, drop_remainder=True)
-    ds_test = ds_test.map(preprocessing).batch(batch_size, drop_remainder=True)
-
-    return ds_train, ds_test, ds_info
-
-
-def create_model(num_classes, img_size, learning_rate=1e-2):
-    inputs = layers.Input(shape = (img_size, img_size, 3))
-
-    backbone = EfficientNetB0(include_top=False, input_tensor=inputs)
-
+    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
+                                          data_dir='/tmp/data',
+                                          split=['train', 'validation'],
+                                          with_info=True,
+                                          as_supervised=True)
+
+    # Create a Dataset that includes only 1/num_shards of full dataset.
+    num_shards = int(os.environ.get('NUM_SHARDS', 1))
+    train_ds = train_ds.shard(num_shards, index=0)
+    test_ds = test_ds.shard(num_shards, index=0)
+    num_classes = info.features['label'].num_classes
+    
+    train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+                                                tf.one_hot(label, num_classes))).batch(batch_size)
+    test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+                                              tf.one_hot(label, num_classes))).batch(batch_size)
+    return train_ds, test_ds, info
+
+
+def create_model(num_classes, img_size):
+    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
+    x = tf.cast(inputs, tf.float32)
+    x = tf.keras.applications.resnet50.preprocess_input(x)
+    backbone = ResNet50(weights='imagenet')
     backbone.trainable = False
+    x = backbone(x)
+    x = layers.Dense(512, activation='relu')(x)
+    outputs = layers.Dense(num_classes, activation='softmax')(x)
 
-    x = layers.GlobalAveragePooling2D(name='avg_pool')(backbone.output)
-    x = layers.BatchNormalization()(x)
-
-    top_dropout_rate = 0.2
-    x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
-    outputs = layers.Dense(num_classes, activation="softmax", name="pred")(x)
-
-    model = Model(inputs, outputs, name='EfficientNet')
-    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
-    model.compile(
-        loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
-    )
+    model = Model(inputs=inputs, outputs=outputs)
+    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
     return model
 
 
 if __name__ == '__main__':
     img_size = 224
     batch_size = 32
-    num_epochs = int(os.environ.get('NUM_EPOCHS', 10))
-    
-    ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
-    
+
+    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
     num_classes = ds_info.features['label'].num_classes
-    steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
-    validation_steps = ds_info.splits['test'].num_examples // batch_size
 
-    # Multi-Instance Training
-    # 
-    # It is often beneficial to use multiple instances for training
-    # if a server contains multiple sockets or many cores, 
-    # so that the workload can make full use of all CPU cores.
-    # BigDL-Nano makes it very easy to conduct multi-instance training correctly.
+    # Quantization with Intel Neural Compressor
+    #
+    # A quantized model executes tensor operations with reduced precision.
+    # This allows for a more compact model representation and the use of high 
+    # performance vectorized operations on many hardware platforms.
+    # It can be used to speed up inference and only the forward pass is supported
+    # for quantized operators.
     # 
-    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create model,
-    # then just set the `num_processes` parameter in the `fit` method.
-    # BigDL-Nano will launch the specific number of processes to perform data-parallel training.
+    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
+    # `Model.quantize()` return a Keras module with desired precision and accuracy.
+    # Taking Resnet50 as an example, you can add quantization as below.
     #
-    model = create_model(num_classes=num_classes, img_size=img_size)
-    q_model = model.quantize(calib_dataset=ds_test,
+    model = create_model(num_classes, img_size)
+    q_model = model.quantize(calib_dataset=test_ds,
                              metric=CategoricalAccuracy(),
                              tuning_strategy='basic')
-

From 8e956e76acb586d183ec6746d0c58e9c0af74e8f Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Tue, 23 Aug 2022 10:02:00 +0800
Subject: [PATCH 26/33] set jemalloc as the default allocator

---
 python/nano/scripts/bigdl-nano-init | 1 +
 1 file changed, 1 insertion(+)

diff --git a/python/nano/scripts/bigdl-nano-init b/python/nano/scripts/bigdl-nano-init
index b7117fe22f7..f48bc88dbf7 100644
--- a/python/nano/scripts/bigdl-nano-init
+++ b/python/nano/scripts/bigdl-nano-init
@@ -170,6 +170,7 @@ export MALLOC_CONF="oversize_threshold:1,background_thread:true,metadata_thp:aut
 # Set LD_PRELOAD
 if [ -z "${LD_PRELOAD:-}" ]; then
 	echo "Setting LD_PRELOAD..."
+	ENABLE_JEMALLOC_VAR=1
 	if [[ $OPENMP -eq 1 && -z "${DISABLE_OPENMP_VAR:-}" ]]; then
 		export LD_PRELOAD="${LIB_DIR}/libiomp5.so"
 	fi

From b25ddba212c6e3e229d871da48b739394b46c118 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Tue, 23 Aug 2022 13:38:31 +0800
Subject: [PATCH 27/33] fix

---
 python/nano/scripts/bigdl-nano-init | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/python/nano/scripts/bigdl-nano-init b/python/nano/scripts/bigdl-nano-init
index f48bc88dbf7..74dd0f17b6b 100644
--- a/python/nano/scripts/bigdl-nano-init
+++ b/python/nano/scripts/bigdl-nano-init
@@ -47,6 +47,9 @@ function display-help {
 		echo "    -c, --disable-tcmalloc   Disable tcmalloc and unset related variables"
 }
 
+# set the default allocator to jemalloc
+ENABLE_JEMALLOC_VAR=1
+
 while getopts ":ojhc-:" opt; do
 	case ${opt} in
 		- )
@@ -170,7 +173,6 @@ export MALLOC_CONF="oversize_threshold:1,background_thread:true,metadata_thp:aut
 # Set LD_PRELOAD
 if [ -z "${LD_PRELOAD:-}" ]; then
 	echo "Setting LD_PRELOAD..."
-	ENABLE_JEMALLOC_VAR=1
 	if [[ $OPENMP -eq 1 && -z "${DISABLE_OPENMP_VAR:-}" ]]; then
 		export LD_PRELOAD="${LIB_DIR}/libiomp5.so"
 	fi

From 88fa96b270fb696dfda753c581914d24754b223d Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Tue, 23 Aug 2022 14:00:22 +0800
Subject: [PATCH 28/33] try to fix multi-instance training example

---
 .../tensorflow_train_multi_instance.py        | 43 +++++++++++++------
 1 file changed, 29 insertions(+), 14 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 4cbea087c59..86eda2ba58f 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -20,6 +20,7 @@
 
 import tensorflow as tf
 from tensorflow.keras import layers, Sequential
+from tensorflow.keras.applications import ResNet50
 from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
@@ -56,25 +57,39 @@ def preprocessing(img, label):
     return ds_train, ds_test, ds_info
 
 
-def create_model(num_classes, img_size, learning_rate=1e-2):
-    inputs = layers.Input(shape = (img_size, img_size, 3))
+# def create_model(num_classes, img_size, learning_rate=1e-2):
+#     inputs = layers.Input(shape = (img_size, img_size, 3))
 
-    backbone = EfficientNetB0(include_top=False, input_tensor=inputs)
+#     backbone = EfficientNetB0(include_top=False, input_tensor=inputs)
 
-    backbone.trainable = False
+#     backbone.trainable = False
 
-    x = layers.GlobalAveragePooling2D(name='avg_pool')(backbone.output)
-    x = layers.BatchNormalization()(x)
+#     x = layers.GlobalAveragePooling2D(name='avg_pool')(backbone.output)
+#     x = layers.BatchNormalization()(x)
 
-    top_dropout_rate = 0.2
-    x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
-    outputs = layers.Dense(num_classes, activation="softmax", name="pred")(x)
+#     top_dropout_rate = 0.2
+#     x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
+#     outputs = layers.Dense(num_classes, activation="softmax", name="pred")(x)
 
-    model = Model(inputs, outputs, name='EfficientNet')
-    optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
-    model.compile(
-        loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
-    )
+#     model = Model(inputs, outputs, name='EfficientNet')
+#     optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
+#     model.compile(
+#         loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
+#     )
+#     return model
+
+def create_model(num_classes, img_size):
+    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
+    x = tf.cast(inputs, tf.float32)
+    x = tf.keras.applications.resnet50.preprocess_input(x)
+    backbone = ResNet50()
+    backbone.trainable = False
+    x = backbone(x)
+    x = layers.Dense(512, activation='relu')(x)
+    outputs = layers.Dense(num_classes, activation='softmax')(x)
+
+    model = Model(inputs=inputs, outputs=outputs)
+    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
     return model
 
 

From d323dc82d90162a7bd89b820a1e85eda445cece8 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Tue, 23 Aug 2022 14:41:41 +0800
Subject: [PATCH 29/33] use a smaller dataset in multi-instance training
 example

---
 .../tensorflow/run-nano-tensorflow-test.sh    |  1 +
 .../tensorflow_train_multi_instance.py        | 26 +----
 .../nano/tutorial/training/tensorflow/tmp.py  | 94 +++++++++++++++++++
 3 files changed, 100 insertions(+), 21 deletions(-)
 create mode 100644 python/nano/tutorial/training/tensorflow/tmp.py

diff --git a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
index 5a212ba586e..6e43af0ef6b 100644
--- a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+++ b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
@@ -6,4 +6,5 @@ set -e
 
 export NUM_EPOCHS=1
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_sparse_embedding.py
+export NUM_SHARDS=4
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_train_multi_instance.py
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 86eda2ba58f..8f04a6094b0 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -36,6 +36,11 @@ def create_datasets(img_size, batch_size):
         with_info=True,
         as_supervised=True
     )
+    
+    # Create a Dataset that includes only 1/num_shards of full dataset.
+    num_shards = int(os.environ.get('NUM_SHARDS', 1))
+    ds_train = ds_train.shard(num_shards, index=0)
+    ds_test = ds_test.shard(num_shards, index=0)
 
     num_classes = ds_info.features['label'].num_classes
 
@@ -57,27 +62,6 @@ def preprocessing(img, label):
     return ds_train, ds_test, ds_info
 
 
-# def create_model(num_classes, img_size, learning_rate=1e-2):
-#     inputs = layers.Input(shape = (img_size, img_size, 3))
-
-#     backbone = EfficientNetB0(include_top=False, input_tensor=inputs)
-
-#     backbone.trainable = False
-
-#     x = layers.GlobalAveragePooling2D(name='avg_pool')(backbone.output)
-#     x = layers.BatchNormalization()(x)
-
-#     top_dropout_rate = 0.2
-#     x = layers.Dropout(top_dropout_rate, name="top_dropout")(x)
-#     outputs = layers.Dense(num_classes, activation="softmax", name="pred")(x)
-
-#     model = Model(inputs, outputs, name='EfficientNet')
-#     optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
-#     model.compile(
-#         loss="categorical_crossentropy", optimizer=optimizer, metrics=['accuracy']
-#     )
-#     return model
-
 def create_model(num_classes, img_size):
     inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
     x = tf.cast(inputs, tf.float32)
diff --git a/python/nano/tutorial/training/tensorflow/tmp.py b/python/nano/tutorial/training/tensorflow/tmp.py
new file mode 100644
index 00000000000..7e65c51c1a5
--- /dev/null
+++ b/python/nano/tutorial/training/tensorflow/tmp.py
@@ -0,0 +1,94 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This example shows how to do quantization with bigdl-nano
+
+# Required Dependecies
+
+# ```bash
+# pip install neural-compressor==1.11.0
+# ```
+
+
+import os
+import tensorflow as tf
+from tensorflow.keras import layers
+from tensorflow.keras.applications import ResNet50
+from tensorflow.keras.metrics import CategoricalAccuracy
+import tensorflow_datasets as tfds
+
+# Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
+from bigdl.nano.tf.keras import Model, Sequential
+
+
+def create_datasets(img_size, batch_size):
+    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
+                                          data_dir='/tmp/data',
+                                          split=['train', 'validation'],
+                                          with_info=True,
+                                          as_supervised=True)
+
+    # Create a Dataset that includes only 1/num_shards of full dataset.
+    num_shards = int(os.environ.get('NUM_SHARDS', 1))
+    train_ds = train_ds.shard(num_shards, index=0)
+    test_ds = test_ds.shard(num_shards, index=0)
+    num_classes = info.features['label'].num_classes
+    
+    train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+                                                tf.one_hot(label, num_classes))).batch(batch_size)
+    test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
+                                              tf.one_hot(label, num_classes))).batch(batch_size)
+    return train_ds, test_ds, info
+
+
+def create_model(num_classes, img_size):
+    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
+    x = tf.cast(inputs, tf.float32)
+    x = tf.keras.applications.resnet50.preprocess_input(x)
+    backbone = ResNet50(weights='imagenet')
+    backbone.trainable = False
+    x = backbone(x)
+    x = layers.Dense(512, activation='relu')(x)
+    outputs = layers.Dense(num_classes, activation='softmax')(x)
+
+    model = Model(inputs=inputs, outputs=outputs)
+    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
+    return model
+
+
+if __name__ == '__main__':
+    img_size = 224
+    batch_size = 32
+
+    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
+    num_classes = ds_info.features['label'].num_classes
+
+    # Quantization with Intel Neural Compressor
+    #
+    # A quantized model executes tensor operations with reduced precision.
+    # This allows for a more compact model representation and the use of high 
+    # performance vectorized operations on many hardware platforms.
+    # It can be used to speed up inference and only the forward pass is supported
+    # for quantized operators.
+    # 
+    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
+    # `Model.quantize()` return a Keras module with desired precision and accuracy.
+    # Taking Resnet50 as an example, you can add quantization as below.
+    #
+    model = create_model(num_classes, img_size)
+    # q_model = model.quantize(calib_dataset=test_ds,
+    #                          metric=CategoricalAccuracy(),
+    #                          tuning_strategy='basic')
+    model = model.fit(train_ds, epochs=1, validation_data=test_ds, num_processes=2)

From beaa3aab0b56ccfeabb619ee17b40af0fe1e0b0c Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Tue, 23 Aug 2022 14:42:12 +0800
Subject: [PATCH 30/33] fix

---
 .../nano/tutorial/training/tensorflow/tmp.py  | 94 -------------------
 1 file changed, 94 deletions(-)
 delete mode 100644 python/nano/tutorial/training/tensorflow/tmp.py

diff --git a/python/nano/tutorial/training/tensorflow/tmp.py b/python/nano/tutorial/training/tensorflow/tmp.py
deleted file mode 100644
index 7e65c51c1a5..00000000000
--- a/python/nano/tutorial/training/tensorflow/tmp.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#
-# Copyright 2016 The BigDL Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This example shows how to do quantization with bigdl-nano
-
-# Required Dependecies
-
-# ```bash
-# pip install neural-compressor==1.11.0
-# ```
-
-
-import os
-import tensorflow as tf
-from tensorflow.keras import layers
-from tensorflow.keras.applications import ResNet50
-from tensorflow.keras.metrics import CategoricalAccuracy
-import tensorflow_datasets as tfds
-
-# Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
-from bigdl.nano.tf.keras import Model, Sequential
-
-
-def create_datasets(img_size, batch_size):
-    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
-                                          data_dir='/tmp/data',
-                                          split=['train', 'validation'],
-                                          with_info=True,
-                                          as_supervised=True)
-
-    # Create a Dataset that includes only 1/num_shards of full dataset.
-    num_shards = int(os.environ.get('NUM_SHARDS', 1))
-    train_ds = train_ds.shard(num_shards, index=0)
-    test_ds = test_ds.shard(num_shards, index=0)
-    num_classes = info.features['label'].num_classes
-    
-    train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-                                                tf.one_hot(label, num_classes))).batch(batch_size)
-    test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-                                              tf.one_hot(label, num_classes))).batch(batch_size)
-    return train_ds, test_ds, info
-
-
-def create_model(num_classes, img_size):
-    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
-    x = tf.cast(inputs, tf.float32)
-    x = tf.keras.applications.resnet50.preprocess_input(x)
-    backbone = ResNet50(weights='imagenet')
-    backbone.trainable = False
-    x = backbone(x)
-    x = layers.Dense(512, activation='relu')(x)
-    outputs = layers.Dense(num_classes, activation='softmax')(x)
-
-    model = Model(inputs=inputs, outputs=outputs)
-    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
-    return model
-
-
-if __name__ == '__main__':
-    img_size = 224
-    batch_size = 32
-
-    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
-    num_classes = ds_info.features['label'].num_classes
-
-    # Quantization with Intel Neural Compressor
-    #
-    # A quantized model executes tensor operations with reduced precision.
-    # This allows for a more compact model representation and the use of high 
-    # performance vectorized operations on many hardware platforms.
-    # It can be used to speed up inference and only the forward pass is supported
-    # for quantized operators.
-    # 
-    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
-    # `Model.quantize()` return a Keras module with desired precision and accuracy.
-    # Taking Resnet50 as an example, you can add quantization as below.
-    #
-    model = create_model(num_classes, img_size)
-    # q_model = model.quantize(calib_dataset=test_ds,
-    #                          metric=CategoricalAccuracy(),
-    #                          tuning_strategy='basic')
-    model = model.fit(train_ds, epochs=1, validation_data=test_ds, num_processes=2)

From f1135e59c66a3bb624fbb4542e01b2ab5ad037b5 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Tue, 23 Aug 2022 16:01:37 +0800
Subject: [PATCH 31/33] fix

---
 .github/workflows/nano_notebooks_tests.yml    |  2 +-
 python/nano/scripts/bigdl-nano-init           |  2 +-
 .../tensorflow/run-nano-tensorflow-test.sh    |  1 -
 .../tensorflow_train_multi_instance.py        | 62 +++++++------------
 4 files changed, 23 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index ad96c8963ed..56117eeea41 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -139,8 +139,8 @@ jobs:
           pip install neural-compressor==1.11
           pip install tensorflow-datasets
           source bigdl-nano-init
-          bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
           bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+          bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
           source $CONDA/bin/deactivate
           $CONDA/bin/conda remove -n example-tensorflow --all
         env:
diff --git a/python/nano/scripts/bigdl-nano-init b/python/nano/scripts/bigdl-nano-init
index 74dd0f17b6b..af8315a3a1a 100644
--- a/python/nano/scripts/bigdl-nano-init
+++ b/python/nano/scripts/bigdl-nano-init
@@ -47,7 +47,7 @@ function display-help {
 		echo "    -c, --disable-tcmalloc   Disable tcmalloc and unset related variables"
 }
 
-# set the default allocator to jemalloc
+# set jemalloc as the default allocator
 ENABLE_JEMALLOC_VAR=1
 
 while getopts ":ojhc-:" opt; do
diff --git a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
index 6e43af0ef6b..5a212ba586e 100644
--- a/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+++ b/python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
@@ -6,5 +6,4 @@ set -e
 
 export NUM_EPOCHS=1
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_sparse_embedding.py
-export NUM_SHARDS=4
 python $NANO_TUTORIAL_TEST_DIR/tensorflow_train_multi_instance.py
diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 8f04a6094b0..5ea629915e8 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -17,11 +17,9 @@
 
 
 import os
-
 import tensorflow as tf
-from tensorflow.keras import layers, Sequential
+from tensorflow.keras import layers
 from tensorflow.keras.applications import ResNet50
-from tensorflow.keras.applications import EfficientNetB0
 import tensorflow_datasets as tfds
 
 # Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
@@ -29,44 +27,28 @@
 
 
 def create_datasets(img_size, batch_size):
-    (ds_train, ds_test), ds_info = tfds.load(
-        "stanford_dogs",
-        data_dir="/tmp/data",
-        split=['train', 'test'],
-        with_info=True,
-        as_supervised=True
-    )
+    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
+                                          data_dir='/tmp/data',
+                                          split=['train', 'validation'],
+                                          with_info=True,
+                                          as_supervised=True)
+    
+    num_classes = info.features['label'].num_classes
     
-    # Create a Dataset that includes only 1/num_shards of full dataset.
-    num_shards = int(os.environ.get('NUM_SHARDS', 1))
-    ds_train = ds_train.shard(num_shards, index=0)
-    ds_test = ds_test.shard(num_shards, index=0)
-
-    num_classes = ds_info.features['label'].num_classes
-
-    data_augmentation = Sequential([
-        layers.RandomFlip(),
-        layers.RandomRotation(factor=0.15),
-    ])
-
     def preprocessing(img, label):
-        img, label =  tf.image.resize(img, (img_size, img_size)), tf.one_hot(label, num_classes)
-        return data_augmentation(img), label
+        return tf.image.resize(img, (img_size, img_size)), \
+               tf.one_hot(label, num_classes)
 
-    AUTOTUNE = tf.data.AUTOTUNE
-    ds_train = ds_train.cache().repeat().map(preprocessing). \
-        batch(batch_size, drop_remainder=True).prefetch(AUTOTUNE)
-    ds_test = ds_test.map(preprocessing). \
-        batch(batch_size, drop_remainder=True).prefetch(AUTOTUNE)
-
-    return ds_train, ds_test, ds_info
+    train_ds = train_ds.repeat().map(preprocessing).batch(batch_size)
+    test_ds = test_ds.map(preprocessing).batch(batch_size)
+    return train_ds, test_ds, info
 
 
 def create_model(num_classes, img_size):
     inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
     x = tf.cast(inputs, tf.float32)
     x = tf.keras.applications.resnet50.preprocess_input(x)
-    backbone = ResNet50()
+    backbone = ResNet50(weights='imagenet')
     backbone.trainable = False
     x = backbone(x)
     x = layers.Dense(512, activation='relu')(x)
@@ -81,12 +63,12 @@ def create_model(num_classes, img_size):
     img_size = 224
     batch_size = 32
     num_epochs = int(os.environ.get('NUM_EPOCHS', 10))
-    
-    ds_train, ds_test, ds_info = create_datasets(img_size=img_size, batch_size=batch_size)
-    
+
+    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
+
     num_classes = ds_info.features['label'].num_classes
+
     steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
-    validation_steps = ds_info.splits['test'].num_examples // batch_size
 
     # Multi-Instance Training
     # 
@@ -99,10 +81,8 @@ def create_model(num_classes, img_size):
     # then just set the `num_processes` parameter in the `fit` method.
     # BigDL-Nano will launch the specific number of processes to perform data-parallel training.
     #
-    model = create_model(num_classes=num_classes, img_size=img_size)
-    model.fit(ds_train,
+    model = create_model(num_classes, img_size)
+    model.fit(train_ds,
               epochs=num_epochs,
               steps_per_epoch=steps_per_epoch,
-              validation_data=ds_test,
-              validation_steps=validation_steps,
-              num_processes=4)
+              num_processes=2)

From d946032f1b7fb77533ad771e1194c5032869c205 Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Tue, 6 Sep 2022 16:47:34 +0800
Subject: [PATCH 32/33] remove tensorflow quantization example

---
 .github/workflows/nano_notebooks_tests.yml    |  1 -
 python/nano/scripts/bigdl-nano-init           |  3 -
 .../run-nano-tensorflow-inference-test.sh     |  8 --
 .../tensorflow/tensorflow_quantization.py     | 93 -------------------
 4 files changed, 105 deletions(-)
 delete mode 100644 python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
 delete mode 100644 python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py

diff --git a/.github/workflows/nano_notebooks_tests.yml b/.github/workflows/nano_notebooks_tests.yml
index 56117eeea41..a0e1587a022 100644
--- a/.github/workflows/nano_notebooks_tests.yml
+++ b/.github/workflows/nano_notebooks_tests.yml
@@ -140,7 +140,6 @@ jobs:
           pip install tensorflow-datasets
           source bigdl-nano-init
           bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
-          bash python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
           source $CONDA/bin/deactivate
           $CONDA/bin/conda remove -n example-tensorflow --all
         env:
diff --git a/python/nano/scripts/bigdl-nano-init b/python/nano/scripts/bigdl-nano-init
index af8315a3a1a..b7117fe22f7 100644
--- a/python/nano/scripts/bigdl-nano-init
+++ b/python/nano/scripts/bigdl-nano-init
@@ -47,9 +47,6 @@ function display-help {
 		echo "    -c, --disable-tcmalloc   Disable tcmalloc and unset related variables"
 }
 
-# set jemalloc as the default allocator
-ENABLE_JEMALLOC_VAR=1
-
 while getopts ":ojhc-:" opt; do
 	case ${opt} in
 		- )
diff --git a/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh b/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
deleted file mode 100644
index 998fa6a4c77..00000000000
--- a/python/nano/tutorial/inference/tensorflow/run-nano-tensorflow-inference-test.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-export ANALYTICS_ZOO_ROOT=${ANALYTICS_ZOO_ROOT}
-export NANO_HOME=${ANALYTICS_ZOO_ROOT}/python/nano/src
-export NANO_TUTORIAL_TEST_DIR=${ANALYTICS_ZOO_ROOT}/python/nano/tutorial/inference/tensorflow
-
-set -e
-
-export NUM_SHARDS=4
-python $NANO_TUTORIAL_TEST_DIR/tensorflow_quantization.py
diff --git a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py b/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
deleted file mode 100644
index 80d100f8aca..00000000000
--- a/python/nano/tutorial/inference/tensorflow/tensorflow_quantization.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#
-# Copyright 2016 The BigDL Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This example shows how to do quantization with bigdl-nano
-
-# Required Dependecies
-
-# ```bash
-# pip install neural-compressor==1.11.0
-# ```
-
-
-import os
-import tensorflow as tf
-from tensorflow.keras import layers
-from tensorflow.keras.applications import ResNet50
-from tensorflow.keras.metrics import CategoricalAccuracy
-import tensorflow_datasets as tfds
-
-# Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
-from bigdl.nano.tf.keras import Model, Sequential
-
-
-def create_datasets(img_size, batch_size):
-    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
-                                          data_dir='/tmp/data',
-                                          split=['train', 'validation'],
-                                          with_info=True,
-                                          as_supervised=True)
-
-    # Create a Dataset that includes only 1/num_shards of full dataset.
-    num_shards = int(os.environ.get('NUM_SHARDS', 1))
-    train_ds = train_ds.shard(num_shards, index=0)
-    test_ds = test_ds.shard(num_shards, index=0)
-    num_classes = info.features['label'].num_classes
-    
-    train_ds = train_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-                                                tf.one_hot(label, num_classes))).batch(batch_size)
-    test_ds = test_ds.map(lambda img, label: (tf.image.resize(img, (img_size, img_size)),
-                                              tf.one_hot(label, num_classes))).batch(batch_size)
-    return train_ds, test_ds, info
-
-
-def create_model(num_classes, img_size):
-    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
-    x = tf.cast(inputs, tf.float32)
-    x = tf.keras.applications.resnet50.preprocess_input(x)
-    backbone = ResNet50(weights='imagenet')
-    backbone.trainable = False
-    x = backbone(x)
-    x = layers.Dense(512, activation='relu')(x)
-    outputs = layers.Dense(num_classes, activation='softmax')(x)
-
-    model = Model(inputs=inputs, outputs=outputs)
-    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
-    return model
-
-
-if __name__ == '__main__':
-    img_size = 224
-    batch_size = 32
-
-    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
-    num_classes = ds_info.features['label'].num_classes
-
-    # Quantization with Intel Neural Compressor
-    #
-    # A quantized model executes tensor operations with reduced precision.
-    # This allows for a more compact model representation and the use of high 
-    # performance vectorized operations on many hardware platforms.
-    # It can be used to speed up inference and only the forward pass is supported
-    # for quantized operators.
-    # 
-    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create a model, then
-    # `Model.quantize()` return a Keras module with desired precision and accuracy.
-    # Taking Resnet50 as an example, you can add quantization as below.
-    #
-    model = create_model(num_classes, img_size)
-    q_model = model.quantize(calib_dataset=test_ds,
-                             metric=CategoricalAccuracy(),
-                             tuning_strategy='basic')

From 23eb4559f2b819e2c0bd4ee7049773d18aa0dcdb Mon Sep 17 00:00:00 2001
From: Yishuo Wang <yishuo.wang@intel.com>
Date: Wed, 7 Sep 2022 10:03:25 +0800
Subject: [PATCH 33/33] add learning rate & warmup explanation

---
 .../tensorflow_train_multi_instance.py           | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
index 5ea629915e8..9347852b182 100644
--- a/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
+++ b/python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py
@@ -75,11 +75,23 @@ def create_model(num_classes, img_size):
     # It is often beneficial to use multiple instances for training
     # if a server contains multiple sockets or many cores, 
     # so that the workload can make full use of all CPU cores.
-    # BigDL-Nano makes it very easy to conduct multi-instance training correctly.
+    #
+    # When using data-parallel training, the batch size is equivalent to
+    # becoming n times larger, where n is the number of parallel processes.
+    # We should to scale the learning rate to n times as well to achieve the
+    # same effect as single instance training.
+    # However, scaling the learning rate linearly may lead to poor convergence
+    # at the beginning of training, so we should gradually increase the
+    # learning rate to n times, and this is called 'learning rate warmup'.
+    # 
+    # Fortunately, BigDL-Nano makes it very easy to conduct multi-instance 
+    # training correctly. It will handle all these for you.
     # 
     # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create model,
     # then just set the `num_processes` parameter in the `fit` method.
-    # BigDL-Nano will launch the specific number of processes to perform data-parallel training.
+    # BigDL-Nano will launch the specific number of processes to perform
+    # data-parallel training, in addition, it will automatically apply
+    # learning rate scaling and warmup for your training.
     #
     model = create_model(num_classes, img_size)
     model.fit(train_ds,