Merge pull request intel-analytics#4530 from dding3/moveutils

Move common.utils to dllib.utils

python/bigdl/__init__.py

@@ -14,5 +14,5 @@
 # limitations under the License.
 #
 
-from bigdl.utils.engine import prepare_env
+from bigdl.dllib.utils.engine import prepare_env
 prepare_env()

python/bigdl/dllib/examples/imageframe/inception_validation.py

@@ -14,8 +14,7 @@
 # limitations under the License.
 #
 
-from bigdl.utils.common import *
-
+from bigdl.dllib.utils.common import *
 from bigdl.dllib.feature.transform.vision.image import *
 
 from bigdl.dllib.optim.optimizer import *
@@ -50,11 +49,11 @@ def run(image_path, model_path, batch_size):
     transformed = transformer(raw_image_frame)
     model = Model.loadModel(model_path)
     result = model.evaluate(transformed, int(batch_size), [Top1Accuracy()])
-    print "top1 accuray", result[0]
+    print("top1 accuray", result[0])
 
 if __name__ == "__main__":
     if len(sys.argv) != 3:
-        print "parameters needed : <imagePath> <modelPath> <batchSize>"
+        print("parameters needed : <imagePath> <modelPath> <batchSize>")
     image_path = sys.argv[1]
     model_path = sys.argv[2]
     batch_size = sys.argv[3]

python/bigdl/dllib/examples/keras/imdb_cnn_lstm.py

@@ -72,7 +72,7 @@ def build_keras_model():
     hdf5_path = "/tmp/imdb.h5"
     keras_model.save(hdf5_path)
 
-    from bigdl.utils.common import *
+    from bigdl.dllib.utils.common import *
     from bigdl.dllib.nn.layer import *
     from bigdl.dllib.optim.optimizer import *
     from bigdl.dllib.nn.criterion import *

python/bigdl/dllib/examples/keras/keras_utils.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import *
 
 
 def save_keras_definition(keras_model, path):

python/bigdl/dllib/examples/keras/mnist_cnn.py

@@ -85,7 +85,7 @@ def build_keras_model():
     json_path = "/tmp/lenet.json"
     save_keras_definition(keras_model, json_path)
 
-    from bigdl.utils.common import *
+    from bigdl.dllib.utils.common import *
     from bigdl.dllib.nn.layer import *
     from bigdl.dllib.optim.optimizer import *
     from bigdl.dllib.nn.criterion import *

python/bigdl/dllib/feature/dataset/dataset.py

@@ -15,9 +15,9 @@
 #
 
 import sys
-from bigdl.utils.common import JavaValue
-from bigdl.utils.common import callBigDlFunc
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import JavaValue
+from bigdl.dllib.utils.common import callBigDlFunc
+from bigdl.dllib.utils.common import *
 from bigdl.dllib.feature.transform.vision.image import *
 
 if sys.version >= '3':

python/bigdl/dllib/feature/dataset/transformer.py

@@ -15,7 +15,7 @@
 #
 
 
-from bigdl.utils.common import Sample
+from bigdl.dllib.utils.common import Sample
 
 
 def normalizer(data, mean, std):

python/bigdl/dllib/feature/transform/vision/image.py

@@ -15,9 +15,9 @@
 #
 
 import sys
-from bigdl.utils.common import JavaValue
-from bigdl.utils.common import callBigDlFunc
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import JavaValue
+from bigdl.dllib.utils.common import callBigDlFunc
+from bigdl.dllib.utils.common import *
 
 if sys.version >= '3':
     long = int

python/bigdl/dllib/keras/backend.py

@@ -15,7 +15,7 @@
 #
 
 from bigdl.dllib.keras.optimization import *
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import *
 
 
 class KerasModelWrapper:

python/bigdl/dllib/keras/converter.py

@@ -16,8 +16,8 @@
 
 import numpy as np
 import bigdl.dllib.nn.layer as BLayer
-import bigdl.utils.common as BCommon
-from bigdl.utils.common import get_activation_by_name
+import bigdl.dllib.utils.common as BCommon
+from bigdl.dllib.utils.common import get_activation_by_name
 from keras.models import model_from_json
 from keras.models import Sequential, Model, Layer
 import keras

python/bigdl/dllib/keras/layers/__init__.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import *
 
 init_engine()
 redire_spark_logs()

python/bigdl/dllib/keras/layers/layer.py

@@ -17,7 +17,7 @@
 import sys
 
 from bigdl.dllib.nn.layer import Layer, Node, SharedStaticUtils, Container
-from bigdl.utils.common import callBigDlFunc, JTensor, JavaValue
+from bigdl.dllib.utils.common import callBigDlFunc, JTensor, JavaValue
 
 if sys.version >= '3':
     long = int

python/bigdl/dllib/keras/optimization.py

@@ -18,7 +18,7 @@
 
 import bigdl.dllib.nn.criterion as bcriterion
 import bigdl.dllib.optim.optimizer as boptimizer
-from bigdl.utils.common import to_list
+from bigdl.dllib.utils.common import to_list
 from bigdl.dllib.keras.converter import *
 from keras.objectives import *
 import six

python/bigdl/dllib/models/lenet/lenet5.py

@@ -20,7 +20,7 @@
 from bigdl.dllib.nn.layer import *
 from bigdl.dllib.nn.criterion import *
 from bigdl.dllib.optim.optimizer import *
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import *
 
 
 def build_model(class_num):

python/bigdl/dllib/models/local_lenet/local_lenet.py

@@ -19,7 +19,7 @@
 from bigdl.dllib.models.lenet.lenet5 import build_model
 from bigdl.dllib.nn.criterion import *
 from bigdl.dllib.optim.optimizer import *
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import *
 
 
 def get_mnist(data_type="train", location="/tmp/mnist"):

python/bigdl/dllib/models/ml_pipeline/dl_classifier.py

@@ -1,6 +1,6 @@
 from pyspark.ml.pipeline import Estimator, Model
 from pyspark.ml.param.shared import *
-from bigdl.utils.common import *
+from bigdl.dllib.utils.common import *
 import sys
 
 if sys.version >= '3':

python/bigdl/dllib/models/rnn/rnnexample.py

@@ -24,8 +24,8 @@
 from bigdl.dllib.nn.layer import *
 from bigdl.dllib.nn.criterion import *
 from bigdl.dllib.optim.optimizer import *
-from bigdl.utils.common import *
-from bigdl.utils.common import Sample
+from bigdl.dllib.utils.common import *
+from bigdl.dllib.utils.common import Sample
 
 def download_data(dest_dir):
     TINYSHAKESPEARE_URL = 'https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt'  # noqa

python/bigdl/dllib/models/textclassifier/textclassifier.py

@@ -23,8 +23,8 @@
 from bigdl.dllib.nn.layer import *
 from bigdl.dllib.nn.criterion import *
 from bigdl.dllib.optim.optimizer import *
-from bigdl.utils.common import *
-from bigdl.utils.common import Sample
+from bigdl.dllib.utils.common import *
+from bigdl.dllib.utils.common import Sample
 import datetime as dt
 
 

python/bigdl/dllib/nn/criterion.py

@@ -17,9 +17,9 @@
 
 import sys
 
-from bigdl.utils.common import JavaValue
-from bigdl.utils.common import callBigDlFunc
-from bigdl.utils.common import JTensor
+from bigdl.dllib.utils.common import JavaValue
+from bigdl.dllib.utils.common import callBigDlFunc
+from bigdl.dllib.utils.common import JTensor
 from bigdl.dllib.nn.layer import Layer
 import numpy as np
 
@@ -979,8 +979,8 @@ def _test():
     import doctest
     from pyspark import SparkContext
     from bigdl.dllib.nn import criterion
-    from bigdl.utils.common import init_engine
-    from bigdl.utils.common import create_spark_conf
+    from bigdl.dllib.utils.common import init_engine
+    from bigdl.dllib.utils.common import create_spark_conf
     globs = criterion.__dict__.copy()
     sc = SparkContext(master="local[4]", appName="test criterion",
                       conf=create_spark_conf())

python/bigdl/dllib/nn/initialization_method.py

@@ -16,7 +16,8 @@
 
 import sys
 
-from bigdl.utils.common import JavaValue
+from bigdl.dllib.utils.common import JavaValue
+
 
 if sys.version >= '3':
     long = int

python/bigdl/dllib/nn/layer.py

@@ -21,14 +21,14 @@
 import numpy as np
 import six
 
-from bigdl.utils.common import JTensor
-from bigdl.utils.common import JavaValue
-from bigdl.utils.common import callBigDlFunc
-from bigdl.utils.common import callJavaFunc
-from bigdl.utils.common import get_spark_context
-from bigdl.utils.common import to_list
-from bigdl.utils.common import INTMAX, INTMIN, DOUBLEMAX
-from bigdl.utils.common import get_activation_by_name
+from bigdl.dllib.utils.common import JTensor
+from bigdl.dllib.utils.common import JavaValue
+from bigdl.dllib.utils.common import callBigDlFunc
+from bigdl.dllib.utils.common import callJavaFunc
+from bigdl.dllib.utils.common import get_spark_context
+from bigdl.dllib.utils.common import to_list
+from bigdl.dllib.utils.common import INTMAX, INTMIN, DOUBLEMAX
+from bigdl.dllib.utils.common import get_activation_by_name
 from bigdl.dllib.optim.optimizer import L1Regularizer, L2Regularizer, L1L2Regularizer
 from py4j.java_gateway import JavaObject
 from pyspark.rdd import RDD
@@ -745,7 +745,7 @@ def __init__(self,
                 bigdl_type, "createModelPreprocessor", inputs, outputs)
             self.bigdl_type = bigdl_type
         else:
-            from bigdl.utils.tf_utils import convert
+            from bigdl.dllib.utils.tf_utils import convert
             model = convert(to_list(inputs), to_list(outputs), byte_order, bigdl_type)
             super(Model, self).__init__(model.value, bigdl_type)
 
@@ -863,8 +863,8 @@ def load_tensorflow(path, inputs, outputs, byte_order = "little_endian",
 
     @staticmethod
     def train(output, data, label, opt_method, criterion, batch_size, end_when, session=None, bigdl_type="float"):
-        from bigdl.utils.tf_utils import get_path
-        from bigdl.utils.common import Sample
+        from bigdl.dllib.utils.tf_utils import get_path
+        from bigdl.dllib.utils.common import Sample
         output_name = output.name.split(":")[0]
         path = get_path(output_name, session)
         sc = get_spark_context()
@@ -5784,8 +5784,8 @@ def _test():
     import doctest
     from pyspark import SparkContext
     from bigdl.dllib.nn import layer
-    from bigdl.utils.common import init_engine
-    from bigdl.utils.common import create_spark_conf
+    from bigdl.dllib.utils.common import init_engine
+    from bigdl.dllib.utils.common import create_spark_conf
     globs = layer.__dict__.copy()
     sc = SparkContext(master="local[4]", appName="test layer",
                       conf=create_spark_conf())

python/bigdl/dllib/nn/onnx/layer.py

@@ -17,7 +17,7 @@
 import sys
 import numpy as np
 from bigdl.dllib.nn.layer import Layer
-from bigdl.utils.common import JTensor
+from bigdl.dllib.utils.common import JTensor
 
 if sys.version >= '3':
     long = int

python/bigdl/dllib/optim/optimizer.py

@@ -23,14 +23,14 @@
 from py4j.java_gateway import JavaObject
 from pyspark.rdd import RDD
 
-from bigdl.utils.common import DOUBLEMAX
-from bigdl.utils.common import JTensor
-from bigdl.utils.common import JavaValue
-from bigdl.utils.common import callBigDlFunc
-from bigdl.utils.common import callJavaFunc
-from bigdl.utils.common import get_node_and_core_number
-from bigdl.utils.common import init_engine
-from bigdl.utils.common import to_list
+from bigdl.dllib.utils.common import DOUBLEMAX
+from bigdl.dllib.utils.common import JTensor
+from bigdl.dllib.utils.common import JavaValue
+from bigdl.dllib.utils.common import callBigDlFunc
+from bigdl.dllib.utils.common import callJavaFunc
+from bigdl.dllib.utils.common import get_node_and_core_number
+from bigdl.dllib.utils.common import init_engine
+from bigdl.dllib.utils.common import to_list
 from bigdl.dllib.feature.dataset.dataset import *
 import warnings
 
@@ -1190,8 +1190,8 @@ def _test():
     import doctest
     from pyspark import SparkContext
     from bigdl.dllib.optim import optimizer
-    from bigdl.utils.common import init_engine
-    from bigdl.utils.common import create_spark_conf
+    from bigdl.dllib.utils.common import init_engine
+    from bigdl.dllib.utils.common import create_spark_conf
     globs = optimizer.__dict__.copy()
     sc = SparkContext(master="local[4]", appName="test optimizer",
                       conf=create_spark_conf())

python/bigdl/utils/common.py → python/bigdl/dllib/utils/common.py

@@ -32,7 +32,7 @@
 import threading
 import tempfile
 import traceback
-from bigdl.utils.engine import get_bigdl_classpath, is_spark_below_2_2
+from bigdl.dllib.utils.engine import get_bigdl_classpath, is_spark_below_2_2
 
 INTMAX = 2147483647
 INTMIN = -2147483648
@@ -154,7 +154,7 @@ class JTensor(object):
     A wrapper to easy our work when need to pass or return Tensor to/from Scala.
 
     >>> import numpy as np
-    >>> from bigdl.utils.common import JTensor
+    >>> from bigdl.dllib.utils.common import JTensor
     >>> np.random.seed(123)
     >>>
     """
@@ -189,8 +189,8 @@ def from_ndarray(cls, a_ndarray, bigdl_type="float"):
         Convert a ndarray to a DenseTensor which would be used in Java side.
 
         >>> import numpy as np
-        >>> from bigdl.utils.common import JTensor
-        >>> from bigdl.utils.common import callBigDlFunc
+        >>> from bigdl.dllib.utils.common import JTensor
+        >>> from bigdl.dllib.utils.common import callBigDlFunc
         >>> np.random.seed(123)
         >>> data = np.random.uniform(0, 1, (2, 3)).astype("float32")
         >>> result = JTensor.from_ndarray(data)
@@ -236,8 +236,8 @@ def sparse(cls, a_ndarray, i_ndarray, shape, bigdl_type="float"):
         :param shape     shape as a DenseTensor.
 
         >>> import numpy as np
-        >>> from bigdl.utils.common import JTensor
-        >>> from bigdl.utils.common import callBigDlFunc
+        >>> from bigdl.dllib.utils.common import JTensor
+        >>> from bigdl.dllib.utils.common import callBigDlFunc
         >>> np.random.seed(123)
         >>> data = np.arange(1, 7).astype("float32")
         >>> indices = np.arange(1, 7)
@@ -314,7 +314,7 @@ def from_ndarray(cls, features, labels, bigdl_type="float"):
         :param bigdl_type: "double" or "float"
 
         >>> import numpy as np
-        >>> from bigdl.utils.common import callBigDlFunc
+        >>> from bigdl.dllib.utils.common import callBigDlFunc
         >>> from numpy.testing import assert_allclose
         >>> np.random.seed(123)
         >>> sample = Sample.from_ndarray(np.random.random((2,3)), np.random.random((2,3)))
@@ -495,7 +495,7 @@ def to_sample_rdd(x, y, numSlices=None):
     :return:
     """
     sc = get_spark_context()
-    from bigdl.utils.common import Sample
+    from bigdl.dllib.utils.common import Sample
     x_rdd = sc.parallelize(x, numSlices)
     y_rdd = sc.parallelize(y, numSlices)
     return x_rdd.zip(y_rdd).map(lambda item: Sample.from_ndarray(item[0], item[1]))

python/bigdl/dllib/utils/tf_utils.py

@@ -27,8 +27,8 @@
 from tensorflow.python.framework import importer
 from tensorflow.python.platform import gfile
 from bigdl.dllib.nn.layer import Model
-from bigdl.utils.common import JTensor
-from bigdl.utils.common import callBigDlFunc
+from bigdl.dllib.utils.common import JTensor
+from bigdl.dllib.utils.common import callBigDlFunc
 import os
 
 def get_path(output_name, sess=None):