add callZooFunc and change all callBigDlFunc to callZooFunc (intel-an…

…alytics#1793)

python/orca/src/bigdl/orca/net/graph_net.py

@@ -22,6 +22,7 @@
 from zoo.pipeline.api.keras.base import ZooKerasLayer
 from zoo.pipeline.api.keras.utils import *
 from bigdl.nn.layer import Layer
+from zoo.common.utils import callZooFunc
 
 if sys.version >= '3':
     long = int
@@ -50,10 +51,10 @@ def predict(self, x, batch_per_thread=4, distributed=True):
                      Default is True. In local mode, x must be a Numpy array.
         """
         if isinstance(x, ImageSet) or isinstance(x, TextSet):
-            results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                    self.value,
-                                    x,
-                                    batch_per_thread)
+            results = callZooFunc(self.bigdl_type, "zooPredict",
+                                  self.value,
+                                  x,
+                                  batch_per_thread)
             return ImageSet(results) if isinstance(x, ImageSet) else TextSet(results)
         if distributed:
             if isinstance(x, np.ndarray):
@@ -62,29 +63,29 @@ def predict(self, x, batch_per_thread=4, distributed=True):
                 data_rdd = x
             else:
                 raise TypeError("Unsupported prediction data type: %s" % type(x))
-            results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                    self.value,
-                                    data_rdd,
-                                    batch_per_thread)
+            results = callZooFunc(self.bigdl_type, "zooPredict",
+                                  self.value,
+                                  data_rdd,
+                                  batch_per_thread)
             return results.map(lambda result: Layer.convert_output(result))
         else:
             if isinstance(x, np.ndarray) or isinstance(x, list):
-                results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                        self.value,
-                                        self._to_jtensors(x),
-                                        batch_per_thread)
+                results = callZooFunc(self.bigdl_type, "zooPredict",
+                                      self.value,
+                                      self._to_jtensors(x),
+                                      batch_per_thread)
                 return [Layer.convert_output(result) for result in results]
             else:
                 raise TypeError("Unsupported prediction data type: %s" % type(x))
 
     def flattened_layers(self, include_container=False):
-        jlayers = callBigDlFunc(self.bigdl_type, "getFlattenSubModules", self, include_container)
+        jlayers = callZooFunc(self.bigdl_type, "getFlattenSubModules", self, include_container)
         layers = [Layer.of(jlayer) for jlayer in jlayers]
         return layers
 
     @property
     def layers(self):
-        jlayers = callBigDlFunc(self.bigdl_type, "getSubModules", self)
+        jlayers = callZooFunc(self.bigdl_type, "getSubModules", self)
         layers = [Layer.of(jlayer) for jlayer in jlayers]
         return layers
 
@@ -107,7 +108,7 @@ def new_graph(self, outputs):
         :param outputs: A list of nodes specified
         :return: A graph model
         """
-        value = callBigDlFunc(self.bigdl_type, "newGraph", self.value, outputs)
+        value = callZooFunc(self.bigdl_type, "newGraph", self.value, outputs)
         return self.from_jvalue(value, self.bigdl_type)
 
     def freeze_up_to(self, names):
@@ -118,7 +119,7 @@ def freeze_up_to(self, names):
         :param names: A list of module names to be Freezed
         :return: current graph model
         """
-        callBigDlFunc(self.bigdl_type, "freezeUpTo", self.value, names)
+        callZooFunc(self.bigdl_type, "freezeUpTo", self.value, names)
 
     def unfreeze(self, names=None):
         """
@@ -129,8 +130,8 @@ def unfreeze(self, names=None):
         :param names: list of module names to be unFreezed. Default is None.
         :return: current graph model
         """
-        callBigDlFunc(self.bigdl_type, "unFreeze", self.value, names)
+        callZooFunc(self.bigdl_type, "unFreeze", self.value, names)
 
     def to_keras(self):
-        value = callBigDlFunc(self.bigdl_type, "netToKeras", self.value)
+        value = callZooFunc(self.bigdl_type, "netToKeras", self.value)
         return ZooKerasLayer.of(value, self.bigdl_type)

python/orca/src/bigdl/orca/net/net_load.py

@@ -18,11 +18,10 @@
 import os
 import sys
 
-from bigdl.util.common import callBigDlFunc
+from zoo.common.utils import callZooFunc
 from bigdl.nn.layer import Model as BModel
 from zoo.pipeline.api.net.graph_net import GraphNet
 
-
 if sys.version >= '3':
     long = int
     unicode = str
@@ -32,9 +31,9 @@ class JavaToPython:
     # TODO: Add more mapping here as it only support Model and Sequential for now.
     def __init__(self, jvalue, bigdl_type="float"):
         self.jvaule = jvalue
-        self.jfullname = callBigDlFunc(bigdl_type,
-                                       "getRealClassNameOfJValue",
-                                       jvalue)
+        self.jfullname = callZooFunc(bigdl_type,
+                                     "getRealClassNameOfJValue",
+                                     jvalue)
 
     def get_python_class(self):
         """
@@ -83,7 +82,7 @@ def load_bigdl(model_path, weight_path=None, bigdl_type="float"):
         :param weight_path: The path to the weights of the pre-trained model. Default is None.
         :return: A pre-trained model.
         """
-        jmodel = callBigDlFunc(bigdl_type, "netLoadBigDL", model_path, weight_path)
+        jmodel = callZooFunc(bigdl_type, "netLoadBigDL", model_path, weight_path)
         return GraphNet.from_jvalue(jmodel)
 
     @staticmethod
@@ -98,7 +97,7 @@ def load(model_path, weight_path=None, bigdl_type="float"):
         :param weight_path: The path for pre-trained weights if any. Default is None.
         :return: An Analytics Zoo model.
         """
-        jmodel = callBigDlFunc(bigdl_type, "netLoad", model_path, weight_path)
+        jmodel = callZooFunc(bigdl_type, "netLoad", model_path, weight_path)
         return Net.from_jvalue(jmodel, bigdl_type)
 
     @staticmethod
@@ -109,7 +108,7 @@ def load_torch(path, bigdl_type="float"):
         :param path: The path containing the pre-trained model.
         :return: A pre-trained model.
         """
-        jmodel = callBigDlFunc(bigdl_type, "netLoadTorch", path)
+        jmodel = callZooFunc(bigdl_type, "netLoadTorch", path)
         return GraphNet.from_jvalue(jmodel, bigdl_type)
 
     @staticmethod
@@ -131,10 +130,10 @@ def load_tf(path, inputs=None, outputs=None, byte_order="little_endian",
         if not inputs and not outputs:  # load_tf from exported folder
             if not os.path.isdir(path):
                 raise ValueError("load_tf from exported folder requires path to be a folder")
-            jmodel = callBigDlFunc(bigdl_type, "netLoadTF", path)
+            jmodel = callZooFunc(bigdl_type, "netLoadTF", path)
         else:
-            jmodel = callBigDlFunc(bigdl_type, "netLoadTF", path, inputs, outputs,
-                                   byte_order, bin_file)
+            jmodel = callZooFunc(bigdl_type, "netLoadTF", path, inputs, outputs,
+                                 byte_order, bin_file)
         return GraphNet.from_jvalue(jmodel, bigdl_type)
 
     @staticmethod
@@ -146,7 +145,7 @@ def load_caffe(def_path, model_path, bigdl_type="float"):
         :param model_path: The path containing the pre-trained caffe model.
         :return: A pre-trained model.
         """
-        jmodel = callBigDlFunc(bigdl_type, "netLoadCaffe", def_path, model_path)
+        jmodel = callZooFunc(bigdl_type, "netLoadCaffe", def_path, model_path)
         return GraphNet.from_jvalue(jmodel, bigdl_type)
 
     @staticmethod

python/orca/src/bigdl/orca/net/tf_dataset.py

@@ -19,7 +19,8 @@
 
 from bigdl.dataset.dataset import DataSet
 from bigdl.transform.vision.image import FeatureTransformer
-from bigdl.util.common import get_node_and_core_number, callBigDlFunc
+from bigdl.util.common import get_node_and_core_number
+from zoo.common.utils import callZooFunc
 from zoo.common import Sample, JTensor
 from zoo.common.nncontext import getOrCreateSparkContext
 from zoo.feature.common import FeatureSet
@@ -556,19 +557,19 @@ def __init__(self, file_path, parse_fn, batch_size,
         serialized_graph = bytearray(g.as_graph_def().SerializeToString())
 
         sc = getOrCreateSparkContext()
-        train_rdd = callBigDlFunc("float", "createRDDFromTFRecords",
-                                  file_path, sc, serialized_graph,
-                                  serialized_example.name, output_names)
+        train_rdd = callZooFunc("float", "createRDDFromTFRecords",
+                                file_path, sc, serialized_graph,
+                                serialized_example.name, output_names)
         validation_rdd = None
         if validation_file_path is not None:
-            validation_rdd = callBigDlFunc("float", "createRDDFromTFRecords",
-                                           validation_file_path, sc, serialized_graph,
-                                           serialized_example.name, output_names)
+            validation_rdd = callZooFunc("float", "createRDDFromTFRecords",
+                                         validation_file_path, sc, serialized_graph,
+                                         serialized_example.name, output_names)
 
         tensor_structure = nest.pack_sequence_as(results,
                                                  [TensorMeta(tf.as_dtype(t.dtype),
-                                                  shape=t.shape,
-                                                  name="data_%s" % i)
+                                                             shape=t.shape,
+                                                             name="data_%s" % i)
                                                   for i, t in enumerate(nest.flatten(results))])
 
         super(TFRecordDataset, self).__init__(tensor_structure, batch_size,

python/orca/src/bigdl/orca/net/tf_optimizer.py

@@ -22,7 +22,8 @@
 
 from bigdl.nn.criterion import Criterion
 from bigdl.nn.layer import Layer
-from bigdl.util.common import to_list, JavaValue, callBigDlFunc
+from bigdl.util.common import to_list, JavaValue
+from zoo.common.utils import callZooFunc
 from bigdl.optim.optimizer import MaxEpoch, EveryEpoch
 from zoo.pipeline.api.keras.engine.topology import to_bigdl_metric, Loss
 from zoo.pipeline.api.net.utils import _find_placeholders, to_bigdl_optim_method
@@ -88,8 +89,8 @@ def __init__(self, path, config_proto, saver, meta, sess):
         super(TFTrainingHelper2, self).__init__(None, "float", path, byte_arr)
 
     def save_checkpoint(self):
-        callBigDlFunc(self.bigdl_type, "saveCheckpoint",
-                      self.value)
+        callZooFunc(self.bigdl_type, "saveCheckpoint",
+                    self.value)
 
     def get_weights_to_python(self):
         self.save_checkpoint()

python/orca/src/bigdl/orca/net/tfnet.py

@@ -26,7 +26,7 @@
 from zoo.common.nncontext import getOrCreateSparkContext
 from zoo.common import JTensor, Sample
 from zoo.feature.image import ImageSet
-from bigdl.util.common import callBigDlFunc
+from zoo.common.utils import callZooFunc
 
 if sys.version >= '3':
     long = int
@@ -107,10 +107,10 @@ def predict(self, x, batch_per_thread=1, distributed=True):
         Use a model to do prediction.
         """
         if isinstance(x, ImageSet):
-            results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                    self.value,
-                                    x,
-                                    batch_per_thread)
+            results = callZooFunc(self.bigdl_type, "zooPredict",
+                                  self.value,
+                                  x,
+                                  batch_per_thread)
             return ImageSet(results)
         if distributed:
             if isinstance(x, np.ndarray):
@@ -119,10 +119,10 @@ def predict(self, x, batch_per_thread=1, distributed=True):
                 data_rdd = x
             else:
                 raise TypeError("Unsupported prediction data type: %s" % type(x))
-            results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                    self.value,
-                                    data_rdd,
-                                    batch_per_thread)
+            results = callZooFunc(self.bigdl_type, "zooPredict",
+                                  self.value,
+                                  data_rdd,
+                                  batch_per_thread)
             return results.map(lambda result: Layer.convert_output(result))
         else:
             start_idx = 0

python/orca/src/bigdl/orca/net/torch_net.py

@@ -24,7 +24,7 @@
 from bigdl.nn.layer import Layer
 from zoo import getOrCreateSparkContext
 from zoo.feature.image import ImageSet
-from bigdl.util.common import callBigDlFunc
+from zoo.common.utils import callZooFunc
 from zoo.pipeline.api.net.tfnet import to_sample_rdd
 
 if sys.version >= '3':
@@ -97,18 +97,18 @@ def savePytorch(self, path):
         save the model as a torch script module
         '''
         pythonBigDL_method_name = "torchNetSavePytorch"
-        callBigDlFunc(self.bigdl_type, pythonBigDL_method_name, self.value, path)
+        callZooFunc(self.bigdl_type, pythonBigDL_method_name, self.value, path)
         return
 
     def predict(self, x, batch_per_thread=1, distributed=True):
         """
         Use a model to do prediction.
         """
         if isinstance(x, ImageSet):
-            results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                    self.value,
-                                    x,
-                                    batch_per_thread)
+            results = callZooFunc(self.bigdl_type, "zooPredict",
+                                  self.value,
+                                  x,
+                                  batch_per_thread)
             return ImageSet(results)
         if distributed:
             if isinstance(x, np.ndarray):
@@ -117,17 +117,17 @@ def predict(self, x, batch_per_thread=1, distributed=True):
                 data_rdd = x
             else:
                 raise TypeError("Unsupported prediction data type: %s" % type(x))
-            results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                    self.value,
-                                    data_rdd,
-                                    batch_per_thread)
+            results = callZooFunc(self.bigdl_type, "zooPredict",
+                                  self.value,
+                                  data_rdd,
+                                  batch_per_thread)
             return results.map(lambda result: Layer.convert_output(result))
         else:
             if isinstance(x, np.ndarray) or isinstance(x, list):
-                results = callBigDlFunc(self.bigdl_type, "zooPredict",
-                                        self.value,
-                                        self._to_jtensors(x),
-                                        batch_per_thread)
+                results = callZooFunc(self.bigdl_type, "zooPredict",
+                                      self.value,
+                                      self._to_jtensors(x),
+                                      batch_per_thread)
                 return [Layer.convert_output(result) for result in results]
             else:
                 raise TypeError("Unsupported prediction data type: %s" % type(x))