add conda env to orca test actions (intel-analytics#5735)

* add conda env to orca test actions * add pull_request * update env * update * update * update * update * update * update cmake * update ut * update ut * update requirements horovod pytorch * update * update * update protobuf * update horovod test * add tensorflow * update tensorflow * update * update python exampletet * update python UT * update * update * update
ForJadeForest · Sep 20, 2022 · eb86129 · eb86129
1 parent 1140f6e
commit eb86129
Show file tree

Hide file tree

Showing 13 changed files with 225 additions and 157 deletions.
diff --git a/.github/actions/orca-jep-exampletest-action/action.yml b/.github/actions/orca-jep-exampletest-action/action.yml
diff --git a/.github/actions/orca-python-py37-spark3-action/action.yml b/.github/actions/orca-python-py37-spark3-action/action.yml
diff --git a/.github/actions/orca/orca-jep-exampletest-action/nightly-test/action.yml b/.github/actions/orca/orca-jep-exampletest-action/nightly-test/action.yml
@@ -0,0 +1,24 @@
+name: 'Run Orca Jep ExampleTest'
+description: 'Run Orca Jep ExampleTest'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-tests-jep.sh
+        chmod a+x apps/run-app-tests-pytorch-pip.sh
+        if [ -f /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth ]
+        then
+          rm /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth
+        fi
+        wget -P /root/.cache/torch/hub/checkpoints/ $HTTP_URI/analytics-zoo-models/face-generation/celebaHQ16_december_s7_i96000-9c72988c.pth
+
+        bash python/orca/dev/example/run-example-tests-jep.sh
+        bash apps/run-app-tests-pytorch-pip.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/orca/orca-python-exampletest-action/nightly-test/action.yml b/.github/actions/orca/orca-python-exampletest-action/nightly-test/action.yml
@@ -0,0 +1,18 @@
+name: 'Run Orca Python ExampleTest'
+description: 'Run Orca Python ExampleTest'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export MAVEN_OPTS="-XX:MaxPermSize=3G -Xmx100G -XX:ReservedCodeCacheSize=512m"
+        export _JAVA_OPTIONS="-Xmx100G -XX:MaxPermSize=3G"
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-tests.sh
+        python/orca/dev/example/run-example-tests.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/orca/orca-python-exampletest-ray-action/nightly-test/action.yml b/.github/actions/orca/orca-python-exampletest-ray-action/nightly-test/action.yml
@@ -0,0 +1,16 @@
+name: 'Run Orca Python ExampleTest Ray'
+description: 'Run Orca Python ExampleTest Ray'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-test-ray.sh
+        python/orca/dev/example/run-example-test-ray.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/orca/orca-python-horovod-pytorch-py37-spark3-action/nightly-test/action.yml b/.github/actions/orca/orca-python-horovod-pytorch-py37-spark3-action/nightly-test/action.yml
@@ -0,0 +1,17 @@
+name: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+description: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/test/run-pytests-horovod-tf.sh
+        python/orca/dev/test/run-pytests-horovod-tf.sh
+        ray stop
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/orca/orca-python-py37-spark3-action/nightly-test/action.yml b/.github/actions/orca/orca-python-py37-spark3-action/nightly-test/action.yml
@@ -0,0 +1,25 @@
+name: 'Run Orca Python Py37 Spark3'
+description: 'Run Orca Python Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        sed -i "s/'bigdl-core=='+VERSION/'bigdl-core==2.1.0b20220811'/g" python/dllib/src/setup.py
+        bash python/dev/release_default_linux_spark312.sh default false false false -Ddata-store-url=$HTTP_URI -U
+        
+        # install dllib
+        pip install python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl
+
+        # install orca
+        pip install python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl
+
+        chmod a+x python/orca/dev/test/run-pytests-spark.sh
+        python/orca/dev/test/run-pytests-spark.sh
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/orca/orca-python-ray-py37-spark3-action/nightly-test/action.yml b/.github/actions/orca/orca-python-ray-py37-spark3-action/nightly-test/action.yml
@@ -0,0 +1,16 @@
+name: 'Run Orca Python Ray Py37 Spark3'
+description: 'Run Orca Python Ray Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/test/run-pytests-ray
+        python/orca/dev/test/run-pytests-ray
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/...-python-exampletest-ray-action/action.yml → ...env/setup-jep-exampletest-py37/action.yml b/...-python-exampletest-ray-action/action.yml → ...env/setup-jep-exampletest-py37/action.yml
@@ -1,37 +1,39 @@
-name: 'Run Orca Python ExampleTest Ray'
-description: 'Run Orca Python ExampleTest Ray'
+name: 'Run Orca Python Jep Exampletest Py37'
+description: 'Run Orca Python Jep Exampletest Py37'
 runs:
   using: "composite"
   steps:
-    - name: Setup Env
-      shell: bash
-      run: |
-        apt-get update
-        apt-get install wget
-
     - name: Set up Python 
       uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: '3.7.10'
 
     - name: Install dependencies
       shell: bash
       run: |
+        apt-get update
+        apt-get install wget
         python -m pip install --upgrade pip
         python -m pip install --upgrade setuptools==58.0.4
-        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
-        pip uninstall -r python/orca/dev/example/requirements-ray.txt -y
-        pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3
-        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt 
-        pip uninstall -y opencv-python-headless
-        pip install opencv-python-headless
 
-    - name: Run Test
+    - name: Setup env
       shell: bash
       run: |
-        export SPARK_LOCAL_HOSTNAME=localhost
-        chmod a+x python/orca/dev/example/run-example-test-ray.sh
-        python/orca/dev/example/run-example-test-ray.sh
+        if conda info --env | grep "py37"; then
+            source activate py37
+        else
+            conda create -n py37 -y python==3.7.10 setuptools==58.0.4
+            conda info --env
+            source activate py37
+        fi
+        
+        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
+        pip uninstall -r python/orca/dev/example/requirements-jep.txt -y
+        pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca
+        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-jep.txt
+        pip uninstall -y opencv-python-headless
+        pip install opencv-python-headless
+        
       env:
         BIGDL_ROOT: ${{ github.workspace }}
-        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/...orca-python-exampletest-action/action.yml → ...etup-orca-horovod-pytorch-py37/action.yml b/...orca-python-exampletest-action/action.yml → ...etup-orca-horovod-pytorch-py37/action.yml
@@ -1,38 +1,40 @@
-name: 'Run Orca Python ExampleTest'
-description: 'Run Orca Python ExampleTest'
+name: 'Run Orca Python Horovod Pytorch Py37 Spark3'
+description: 'Run Orca Python Horovod Pytorch Py37 Spark3'
 runs:
   using: "composite"
   steps:
-    - name: Setup Env
-      shell: bash
-      run: |
-        apt-get update
-        apt-get install wget
-
     - name: Set up Python 
       uses: actions/setup-python@v2
       with:
-        python-version: '3.7.6'
+        python-version: '3.7.10'
 
     - name: Install dependencies
       shell: bash
       run: |
+        apt-get update
+        apt-get install wget
         python -m pip install --upgrade pip
         python -m pip install --upgrade setuptools==58.0.4
+
+    - name: Setup env
+      shell: bash
+      run: |
+        conda create -n py37 -y python==3.7.10 setuptools==58.0.4
+        conda info --env
+        source activate py37
         pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
-        pip uninstall -r python/orca/dev/example/requirements-ray.txt -y
+        conda install -y pytorch==1.4.0 torchvision==0.5.0 cpuonly -c pytorch
+        pip install cmake
+        #pip install tensorflow==2.1
+        HOROVOD_WITH_PYTORCH=1; HOROVOD_WITH_GLOO=1; pip install --no-cache-dir horovod==0.19.2
+        pip uninstall -r .github/actions/python-requirements/requirements-orca-horovod-pytorch.txt -y
+        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r .github/actions/python-requirements/requirements-orca-horovod-pytorch.txt 
         pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3
-        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt 
+        pip install ray[default]==1.9.1
+        pip install protobuf==3.12.0
         pip uninstall -y opencv-python-headless
         pip install opencv-python-headless
-    - name: Run Test
-      shell: bash
-      run: |
-        export MAVEN_OPTS="-XX:MaxPermSize=3G -Xmx100G -XX:ReservedCodeCacheSize=512m"
-        export _JAVA_OPTIONS="-Xmx100G -XX:MaxPermSize=3G"
-        export SPARK_LOCAL_HOSTNAME=localhost
-        chmod a+x python/orca/dev/example/run-example-tests.sh
-        python/orca/dev/example/run-example-tests.sh
+        
       env:
         BIGDL_ROOT: ${{ github.workspace }}
-        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/...-python-ray-py37-spark3-action/action.yml → .../setup-orca-python-py37-spark3/action.yml b/...-python-ray-py37-spark3-action/action.yml → .../setup-orca-python-py37-spark3/action.yml
@@ -1,37 +1,35 @@
-name: 'Run Orca Python Ray Py37 Spark3'
-description: 'Run Orca Python Ray Py37 Spark3'
+name: 'Run Orca Python Py37 Spark3'
+description: 'Run Orca Python Py37 Spark3'
 runs:
   using: "composite"
   steps:
-    - name: Setup Env
-      shell: bash
-      run: |
-        apt-get update
-        apt-get install wget
-
     - name: Set up Python 
       uses: actions/setup-python@v2
       with:
-        python-version: '3.7'
+        python-version: '3.7.10'
 
     - name: Install dependencies
       shell: bash
       run: |
+        apt-get update
+        apt-get install wget
         python -m pip install --upgrade pip
         python -m pip install --upgrade setuptools==58.0.4
+
+    - name: Setup env
+      shell: bash
+      run: |
+        conda create -n py37 -y python==3.7.10 setuptools==58.0.4
+        conda info --env
+        source activate py37
         pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
         pip uninstall -r python/orca/dev/example/requirements-ray.txt -y
+        pip install pytest
         pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3
         pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt 
-        pip install pytest
         pip uninstall -y opencv-python-headless
         pip install opencv-python-headless
-    - name: Run Test
-      shell: bash
-      run: |
-        export SPARK_LOCAL_HOSTNAME=localhost
-        chmod a+x python/orca/dev/test/run-pytests-ray
-        python/orca/dev/test/run-pytests-ray
+        
       env:
         BIGDL_ROOT: ${{ github.workspace }}
         ANALYTICS_ZOO_ROOT: ${{ github.workspace }}