Merge pull request intel-analytics#4546 from dding3/updatepythonstruc…

…ture

Update python structure

python/python-zip.xml → python/dllib/src/python-zip.xml

@@ -11,11 +11,11 @@
             <includes>
                 <include>**/*.py</include>
             </includes>
-            <excludes>
-                <exclude>test/**</exclude>
-            </excludes>
+            <!--<excludes>-->
+                <!--<exclude>test/**</exclude>-->
+            <!--</excludes>-->
             <outputDirectory>/..</outputDirectory>
-            <directory>${project.build.directory}/../../../python</directory>
+            <directory>${project.build.directory}/../../../python/dllib/src/</directory>
         </fileSet>
         <!--<fileSet>-->
             <!--<includes>-->

python/setup.py → python/dllib/src/setup.py

@@ -22,8 +22,8 @@
 
 from setuptools import setup
 
-TEMP_PATH = "bigdl/share"
-bigdl_home = os.path.abspath(__file__ + "/../../")
+TEMP_PATH = "bigdl/dllib/share"
+bigdl_home = os.path.abspath(__file__ + "/../../../..")
 
 try:
     exec(open('bigdl/version.py').read())
@@ -45,7 +45,7 @@
       pip install dist/*.tar.gz"""
 
 def build_from_source():
-    code_path = bigdl_home + "/python/bigdl/utils/common.py"
+    code_path = bigdl_home + "/python/dllib/src/bigdl/dllib/utils/common.py"
     print("Checking: %s to see if build from source" % code_path)
     if os.path.exists(code_path):
         return True
@@ -63,13 +63,13 @@ def init_env():
         if os.path.exists(TEMP_PATH):
             rmtree(TEMP_PATH)
         copytree(dist_source, TEMP_PATH)
-        copyfile(bigdl_home + "/python/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py")
+        copyfile(bigdl_home + "/python/dllib/src/bigdl/dllib/nn/__init__.py", TEMP_PATH + "/__init__.py")
     else:
         print("Do nothing for release installation")
 
 def get_bigdl_packages():
     bigdl_python_home = os.path.abspath(__file__)[:-8]
-    bigdl_packages = ['bigdl.share']
+    bigdl_packages = ['bigdl.dllib.share']
     for dirpath, dirs, files in os.walk(bigdl_python_home + "bigdl"):
         package = dirpath.split(bigdl_python_home)[1].replace('/', '.')
         if "__pycache__" not in package:
@@ -81,7 +81,7 @@ def get_bigdl_packages():
 
 def setup_package():
     metadata = dict(
-        name='BigDL',
+        name='BigDL-dllib',
         version=VERSION,
         description='Distributed Deep Learning Library for Apache Spark',
         author='BigDL Authors',
@@ -93,7 +93,7 @@ def setup_package():
         dependency_links=['https://d3kbcqa49mib13.cloudfront.net/spark-2.0.0-bin-hadoop2.7.tgz'],
         include_package_data=True,
         # package_data={"bigdl.share": ['bigdl/share/lib', 'bigdl/share/conf', 'bigdl/share/bin']},
-        package_data={"bigdl.share": ['bigdl/share/lib']},
+        package_data={"bigdl.dllib.share": ['bigdl/dllib/share/lib']},
         classifiers=[
             'License :: OSI Approved :: Apache Software License',
             'Programming Language :: Python :: 2.7',

python/test/dev/prepare_env.sh → python/dllib/test/dev/prepare_env.sh

@@ -16,9 +16,9 @@
 
 SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]})
 echo "SCRIPT_DIR": $SCRIPT_DIR
-export DL_PYTHON_HOME="$(cd ${SCRIPT_DIR}/../../; pwd)"
+export DL_PYTHON_HOME="$(cd ${SCRIPT_DIR}/../../src; pwd)"
 
-export BIGDL_HOME="$(cd ${SCRIPT_DIR}/../../..; pwd)"
+export BIGDL_HOME="$(cd ${SCRIPT_DIR}/../../../..; pwd)"
 
 echo "BIGDL_HOME: $BIGDL_HOME"
 echo "SPARK_HOME": $SPARK_HOME

python/test/dev/release/release.sh → python/dllib/test/dev/release/release.sh

@@ -19,9 +19,9 @@
 set -e
 RUN_SCRIPT_DIR=$(cd $(dirname $0) ; pwd)
 echo $RUN_SCRIPT_DIR
-BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../..; pwd)"
+BIGDL_DIR="$(cd ${RUN_SCRIPT_DIR}/../../../../..; pwd)"
 echo $BIGDL_DIR
-BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python; pwd)"
+BIGDL_PYTHON_DIR="$(cd ${BIGDL_DIR}/python/dllib/src; pwd)"
 echo $BIGDL_PYTHON_DIR
 
 if (( $# < 2)); then
@@ -32,7 +32,7 @@ fi
 platform=$1
 spark_profile=$2
 quick=$3
-bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/bigdl/version.py').read()); print(__version__)")
+bigdl_version=$(python -c "exec(open('$BIGDL_DIR/python/dllib/src/bigdl/version.py').read()); print(__version__)")
 
 cd ${BIGDL_DIR}/scala
 if [ "$platform" ==  "mac" ]; then
@@ -61,12 +61,12 @@ sdist_command="python setup.py sdist"
 echo "packing source code: ${sdist_command}"
 $sdist_command
 
-if [ -d "${BIGDL_DIR}/python/build" ]; then
-   rm -r ${BIGDL_DIR}/python/build
+if [ -d "${BIGDL_DIR}/python/dllib/build" ]; then
+   rm -r ${BIGDL_DIR}/python/dllib/build
 fi
 
-if [ -d "${BIGDL_DIR}/python/dist" ]; then
-   rm -r ${BIGDL_DIR}/python/dist
+if [ -d "${BIGDL_DIR}/python/dllib/dist" ]; then
+   rm -r ${BIGDL_DIR}/python/dllib/dist
 fi
 
 wheel_command="python setup.py bdist_wheel --plat-name ${verbose_pname}"

python/test/dev/run-caffe.sh → python/dllib/test/dev/run-caffe.sh

@@ -28,8 +28,8 @@ echo "${cyan}Using python version: $p${reset}"
 export PYTHON_EXECUTABLE=$p
 export PYSPARK_PYTHON=$p
 export PYSPARK_DRIVER_PYTHON=$p
-$p -m pytest -v  ../../../python/test/bigdl/caffe/ \
---ignore=../../../python/test/bigdl/caffe/caffe_layers.py
+$p -m pytest -v  ../../../python/dllib/test/bigdl/caffe/ \
+--ignore=../../../python/dllib/test/bigdl/caffe/caffe_layers.py
 exit_status=$?
 echo "running caffe layer unit tests"
 if [ $exit_status -ne 0 ];

python/test/dev/run-keras.sh → python/dllib/test/dev/run-keras.sh

@@ -32,7 +32,7 @@ echo "${cyan}Using python version: $p${reset}"
 export PYTHON_EXECUTABLE=$p
 export PYSPARK_PYTHON=$p
 export PYSPARK_DRIVER_PYTHON=$p
-$p -m pytest -v  ../../../python/test/bigdl/keras
+$p -m pytest -v  ../../../python/dllib/test/bigdl/keras
 
 exit_status=$?
 if [ $exit_status -ne 0 ];

python/test/dev/run-tests → python/dllib/test/dev/run-tests

@@ -33,17 +33,17 @@ do
     export PYTHON_EXECUTABLE=$p
     export PYSPARK_PYTHON=$p
     export PYSPARK_DRIVER_PYTHON=$p
-    $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../python/bigdl \
-    --ignore=../../../python/bigdl/dllib/feature/dataset/ \
-    --ignore=../../../python/bigdl/dllib/utils/tf_utils.py \
-    --ignore=../../../python/bigdl/dllib/keras \
-    --ignore=../../../python/test/bigdl/keras/test_application.py \
-    --ignore=../../../python/bigdl/dllib/examples/ \
-    --ignore=../../../python/bigdl/dllib/models/ && \
+    $p -m pytest -v --junitxml result_bigdl_${p}.xml --doctest-modules ../../../dllib/src/bigdl \
+    --ignore=../../../dllib/src/bigdl/dllib/feature/dataset/ \
+    --ignore=../../../dllib/src/bigdl/dllib/utils/tf_utils.py \
+    --ignore=../../../dllib/src/bigdl/dllib/keras \
+    --ignore=../../../dllib/test/bigdl/keras/test_application.py \
+    --ignore=../../../dllib/src/bigdl/dllib/examples/ \
+    --ignore=../../../dllib/src/bigdl/dllib/models/ && \
     $p -m pytest -v --junitxml result_test_${p}.xml  ../../../python/test/ \
-    --ignore=../../../python/test/bigdl/caffe/ \
-    --ignore=../../../python/test/bigdl/keras/ \
-    --ignore=../../../python/test/bigdl/test_utils.py
+    --ignore=../../../dllib/test/bigdl/caffe/ \
+    --ignore=../../../dllib/test/bigdl/keras/ \
+    --ignore=../../../dllib/test/bigdl/test_utils.py
 
     exit_status=$?
     if [ $exit_status -ne 0 ];

scala/dllib/pom.xml

@@ -261,7 +261,7 @@
                         <inherited>false</inherited>
                         <configuration>
                             <descriptors>
-                                <descriptor>${project.basedir}/../../python/python-zip.xml
+                                <descriptor>${project.basedir}/../../python/dllib/src/python-zip.xml
                                 </descriptor>
                             </descriptors>
                         </configuration>