Add nano tensorflow examples

.github/workflows/nano_notebooks_tests.yml

@@ -131,6 +131,20 @@ jobs:
           python -m pip install --upgrade setuptools==58.0.4
           python -m pip install --upgrade wheel
 
+      - name: Run TensorFlow Example
+        run: |
+          $CONDA/bin/conda create -n example-tensorflow -y python==3.7.10 setuptools=58.0.4
+          source $CONDA/bin/activate example-tensorflow
+          bash python/nano/dev/build_and_install.sh linux default false tensorflow
+          pip install neural-compressor==1.11
+          pip install tensorflow-datasets
+          source bigdl-nano-init
+          bash python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh
+          source $CONDA/bin/deactivate
+          $CONDA/bin/conda remove -n example-tensorflow --all
+        env:
+            ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
+
       - name: Run tutorial notebooks TensorFlow unit tests
         shell: bash
         run: |

python/nano/tutorial/training/tensorflow/run-nano-tensorflow-test.sh

@@ -0,0 +1,9 @@
+export ANALYTICS_ZOO_ROOT=${ANALYTICS_ZOO_ROOT}
+export NANO_HOME=${ANALYTICS_ZOO_ROOT}/python/nano/src
+export NANO_TUTORIAL_TEST_DIR=${ANALYTICS_ZOO_ROOT}/python/nano/tutorial/training/tensorflow
+
+set -e
+
+export NUM_EPOCHS=1
+python $NANO_TUTORIAL_TEST_DIR/tensorflow_sparse_embedding.py
+python $NANO_TUTORIAL_TEST_DIR/tensorflow_train_multi_instance.py

python/nano/tutorial/training/tensorflow/tensorflow_sparse_embedding.py

@@ -0,0 +1,124 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This example shows how to use sparse adam optimizer and embedding layer with bigdl-nano
+
+
+import os
+import re
+import string
+
+import tensorflow as tf
+import tensorflow_datasets as tfds
+from tensorflow.keras import layers
+from tensorflow.keras.layers import TextVectorization
+
+# import `Embedding`, `SparseAdam` and `Model` from bigdl-nano
+from bigdl.nano.tf.keras.layers import Embedding
+from bigdl.nano.tf.optimizers import SparseAdam
+from bigdl.nano.tf.keras import Model
+
+
+max_features = 20000
+embedding_dim = 128
+
+
+def create_datasets():
+    (raw_train_ds, raw_val_ds, raw_test_ds), info = tfds.load(
+        "imdb_reviews",
+        data_dir="/tmp/data",
+        split=['train[:80%]', 'train[80%:]', 'test'],
+        as_supervised=True,
+        batch_size=32,
+        with_info=True
+    )
+
+    def custom_standardization(input_data):
+        lowercase = tf.strings.lower(input_data)
+        stripped_html = tf.strings.regex_replace(lowercase, "<br />", " ")
+        return tf.strings.regex_replace(
+            stripped_html, f"[{re.escape(string.punctuation)}]", ""
+        )
+
+    vectorize_layer = TextVectorization(
+        standardize=custom_standardization,
+        max_tokens=max_features,
+        output_mode="int",
+        output_sequence_length=500,
+    )
+
+    text_ds = raw_train_ds.map(lambda x, y: x)
+    vectorize_layer.adapt(text_ds)
+
+    def vectorize_text(text, label):
+        text = tf.expand_dims(text, -1)
+        return vectorize_layer(text), label
+
+    # Vectorize the data
+    train_ds = raw_train_ds.map(vectorize_text)
+    val_ds = raw_val_ds.map(vectorize_text)
+    test_ds = raw_test_ds.map(vectorize_text)
+
+    return train_ds, val_ds, test_ds
+
+
+def make_backbone():
+    inputs = tf.keras.Input(shape=(None, embedding_dim))
+    x = layers.Dropout(0.5)(inputs)
+    x = layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(x)
+    x = layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(x)
+    x = layers.GlobalMaxPooling1D()(x)
+    x = layers.Dense(128, activation="relu")(x)
+    x = layers.Dropout(0.5)(x)
+    predictions = layers.Dense(1, activation="sigmoid", name="predictions")(x)
+
+    model = Model(inputs, predictions)
+    return model
+
+
+def make_model():
+    inputs = tf.keras.Input(shape=(None,), dtype="int64")
+
+    # use `Embedding` layer in `bigdl.nano.tf.keras.layers`
+    x = Embedding(max_features, embedding_dim)(inputs)
+
+    predictions = make_backbone()(x)
+    model = Model(inputs, predictions)
+
+    # use `SparseAdam` optimizer in `bigdl.nano.tf.optimizers`
+    model.compile(loss="binary_crossentropy", optimizer=SparseAdam(), metrics=["accuracy"])
+
+    return model
+
+
+if __name__=='__main__':
+    num_epochs = int(os.environ.get('NUM_EPOCHS', 10))
+
+    train_ds, val_ds, test_ds = create_datasets()
+
+    # Use sparse adam optimizer and embedding layer
+    #
+    # Sparse embedding represents the gradient matrix by a sparse tensor and 
+    # only calculating gradients for embedding vectors which will be non zero.
+    # It can be used to speed up and reduce memory usage
+    # 
+    # Use `Embedding` in `bigdl.nano.tf.keras.layers` to create a sparse embedding layer,
+    # then use `SparseAdam` in `bigdl.nano.tf.optimizers` as the model's optimizer.
+    #
+    model = make_model()
+
+    model.fit(train_ds, validation_data=val_ds, epochs=num_epochs)
+
+    his = model.evaluate(test_ds)

python/nano/tutorial/training/tensorflow/tensorflow_train_multi_instance.py

@@ -0,0 +1,100 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This example shows how to do multi-process training with bigdl-nano
+
+
+import os
+import tensorflow as tf
+from tensorflow.keras import layers
+from tensorflow.keras.applications import ResNet50
+import tensorflow_datasets as tfds
+
+# Use `Model` and `Sequential` in `bigdl.nano.tf.keras` instead of tensorflow's
+from bigdl.nano.tf.keras import Model
+
+
+def create_datasets(img_size, batch_size):
+    (train_ds, test_ds), info = tfds.load('imagenette/320px-v2',
+                                          data_dir='/tmp/data',
+                                          split=['train', 'validation'],
+                                          with_info=True,
+                                          as_supervised=True)
+
+    num_classes = info.features['label'].num_classes
+
+    def preprocessing(img, label):
+        return tf.image.resize(img, (img_size, img_size)), \
+               tf.one_hot(label, num_classes)
+
+    train_ds = train_ds.repeat().map(preprocessing).batch(batch_size)
+    test_ds = test_ds.map(preprocessing).batch(batch_size)
+    return train_ds, test_ds, info
+
+
+def create_model(num_classes, img_size):
+    inputs = tf.keras.layers.Input(shape=(img_size, img_size, 3))
+    x = tf.cast(inputs, tf.float32)
+    x = tf.keras.applications.resnet50.preprocess_input(x)
+    backbone = ResNet50(weights='imagenet')
+    backbone.trainable = False
+    x = backbone(x)
+    x = layers.Dense(512, activation='relu')(x)
+    outputs = layers.Dense(num_classes, activation='softmax')(x)
+
+    model = Model(inputs=inputs, outputs=outputs)
+    model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=['accuracy'])
+    return model
+
+
+if __name__ == '__main__':
+    img_size = 224
+    batch_size = 32
+    num_epochs = int(os.environ.get('NUM_EPOCHS', 10))
+
+    train_ds, test_ds, ds_info = create_datasets(img_size, batch_size)
+
+    num_classes = ds_info.features['label'].num_classes
+
+    steps_per_epoch = ds_info.splits['train'].num_examples // batch_size
+
+    # Multi-Instance Training
+    # 
+    # It is often beneficial to use multiple instances for training
+    # if a server contains multiple sockets or many cores, 
+    # so that the workload can make full use of all CPU cores.
+    #
+    # When using data-parallel training, the batch size is equivalent to
+    # becoming n times larger, where n is the number of parallel processes.
+    # We should to scale the learning rate to n times as well to achieve the
+    # same effect as single instance training.
+    # However, scaling the learning rate linearly may lead to poor convergence
+    # at the beginning of training, so we should gradually increase the
+    # learning rate to n times, and this is called 'learning rate warmup'.
+    # 
+    # Fortunately, BigDL-Nano makes it very easy to conduct multi-instance 
+    # training correctly. It will handle all these for you.
+    # 
+    # Use `Model` or `Sequential` in `bigdl.nano.tf.keras` to create model,
+    # then just set the `num_processes` parameter in the `fit` method.
+    # BigDL-Nano will launch the specific number of processes to perform
+    # data-parallel training, in addition, it will automatically apply
+    # learning rate scaling and warmup for your training.
+    #
+    model = create_model(num_classes, img_size)
+    model.fit(train_ds,
+              epochs=num_epochs,
+              steps_per_epoch=steps_per_epoch,
+              num_processes=2)