add orca horovod tf2 test to github actions (intel-analytics#5601)

* add orca horovod tf2 test to github actions

* update pull_request

* update

* add keras

* update python version

* update horovod

* update cmake

* install bigdl-orca-spark3

* update

* update mxnet

* update

* update numpy

* update

* update ray version

* unintsall intel-tensorflow

* add conda env

* install horovod

* remove conda env

* install bigdl-orca-spark3

* split conda env

* source conda

* split conda env

* update

* update env

* remove env

* update

* update

* uninstall requirements

* update

* update install whl

* update

* add remove env

* install bigdl-orca-spark3

* update

.github/actions/orca/orca-python-horovod-tf2-py37-spark3-action/nightly-test/action.yml

@@ -0,0 +1,17 @@
+name: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+description: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Run Test
+      shell: bash
+      run: |
+        source activate py37
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/test/run-pytests-horovod-tf.sh
+        python/orca/dev/test/run-pytests-horovod-tf.sh
+        ray stop
+        source deactivate
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/orca/setup-env/setup-horovod-tf2-py37/action.yml

@@ -0,0 +1,50 @@
+name: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+description: 'Run Orca Python Horovod Tf2 Py37 Spark3'
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python 
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7.10'
+
+    - name: Install dependencies
+      shell: bash
+      run: |
+        apt-get update
+        apt-get install wget
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools==58.0.4
+
+    - name: Setup env
+      shell: bash
+      run: |
+        if conda info --env | grep "py37"; then
+            source activate py37
+        else
+            conda create -n py37 -y python==3.7.10 setuptools==58.0.4
+            conda info --env
+            source activate py37
+        fi
+        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
+        pip uninstall -y intel-tensorflow
+        pip install cmake mxnet==1.6.0
+        pip uninstall -r .github/actions/python-requirements/requirements-horovod-tf2.txt -y
+        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r .github/actions/python-requirements/requirements-horovod-tf2.txt 
+        pip install pytest
+        HOROVOD_WITH_PYTORCH=1; HOROVOD_WITH_GLOO=1; pip install --no-cache-dir horovod==0.19.2
+        pip uninstall -y opencv-python-headless
+        pip install opencv-python-headless
+        pip install --pre --upgrade bigdl-orca-spark3
+        #sed -i "s/'bigdl-core=='+VERSION/'bigdl-core==2.1.0b20220811'/g" python/dllib/src/setup.py
+        #bash python/dev/release_default_linux_spark312.sh default false false
+        
+        # install dllib
+        #pip install python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl
+
+        # install orca
+        #pip install python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl
+
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/actions/python-requirements/requirements-horovod-tf2.txt

@@ -0,0 +1,33 @@
+torch==1.7.1
+torchvision
+torchmetrics
+albumentations
+scikit-learn==0.22.2.post1
+opencv-python
+matplotlib
+tqdm
+protobuf==3.15.3
+pandas
+ray[default]==1.9.2
+aiohttp==3.8.1
+async-timeout==4.0.1
+dm_tree
+gym[atari]==0.17.1
+atari-py==0.2.5
+tabulate==0.8.7
+tensorboard==2.3.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.7.0
+tensorboardX==2.1
+tensorflow==2.3.0
+tensorflow-estimator==2.3.0
+xgboost_ray
+argparse
+Pillow
+pyarrow==4.0.1
+h5py==2.10.0
+fsspec==2021.9.0
+scikit-optimize==0.8.1
+pandas==1.1.5
+Keras-Applications==1.0.8
+Keras-Preprocessing==1.1.2

.github/actions/remove-env/action.yml

@@ -0,0 +1,15 @@
+name: 'Remove Env'
+description: 'Remove Env'
+runs:
+  using: "composite"
+  steps:
+    - name: Remove Env
+      shell: bash
+      run: |
+        source activate py37
+        pip uninstall -y pyspark bigdl-friesian bigdl bigdl-dllib-spark3 bigdl-orca-spark3 bigdl-tf bigdl-math bigdl-orca  bigdl-dllib bigdl-tf bigdl-chronos bigdl-nano
+        source deactivate
+        conda remove -n py37 -y --all
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

.github/workflows/nightly_test.yml

@@ -19,6 +19,7 @@ on:
         - Orca-Python-Ray-Py37-Spark3
         - Orca-Python-Py37-Spark3
         - Orca-Ray-Ctx-Example
+        - Orca-Python-Horovod-Tf2-Py37-Spark3
         - Dllib-Scala-UT
         - Friesian-Scala-UT
         - PPML-Scala-UT
@@ -95,6 +96,25 @@ jobs:
     - name: Run test
       uses: ./.github/actions/orca-python-py37-spark3-action
 
+  Orca-Python-Horovod-Tf2-Py37-Spark3:
+    if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-Horovod-Tf2-Py37-Spark3' || github.event.inputs.artifact == 'all' }}
+    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up JDK8
+      uses: ./.github/actions/jdk-setup-action
+    - name: Set up maven
+      uses: ./.github/actions/maven-setup-action 
+    - name: Setup env
+      uses: ./.github/actions/orca/setup-env/setup-horovod-tf2-py37
+    - name: Run Test
+      uses: ./.github/actions/orca/orca-python-horovod-tf2-py37-spark3-action/nightly-test
+    - name: Remove Env
+      if: ${{ always() }}
+      uses: ./.github/actions/remove-env
+
+
   Dllib-Scala-UT:
     if: ${{ github.event.schedule || github.event.inputs.artifact == 'Dllib-Scala-UT' || github.event.inputs.artifact == 'all' }} 
     runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ]