cicd: refactor the github actions workflows (intel-analytics#5529)

* add composite actions * add manually build
ForJadeForest · Sep 20, 2022 · 1476e94 · 1476e94
1 parent f293e44
commit 1476e94
Show file tree

Hide file tree

Showing 9 changed files with 470 additions and 344 deletions.
diff --git a/.github/actions/dllib-scala-ut-action/action.yml b/.github/actions/dllib-scala-ut-action/action.yml
@@ -0,0 +1,21 @@
+name: 'Run Dllib Scala UT'
+description: 'Run Dllib Scala UT'
+runs:
+  using: "composite"
+  steps:
+    - name: Run tests
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        export KERAS_BACKEND=tensorflow
+        cd scala
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.1.2  
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.1.2
+        #need python requirements
+        #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" "-Dtest=com.intel.analytics.bigdl.dllib.keras.*Test" test -P spark_3.x -Dspark.version=3.1.2 
+        #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nn.mkldnn" "-Dtest=com.intel.analytics.bigdl.dllib.nn.mkldnn.*Test" test -P spark_3.x -Dspark.version=3.1.2 
diff --git a/.github/actions/friesian-scala-ut-action/action.yml b/.github/actions/friesian-scala-ut-action/action.yml
@@ -0,0 +1,38 @@
+name: 'Run Friesian Scala UT'
+description: 'Run Friesian Scala UT'
+runs:
+  using: "composite"
+  steps:
+    - name: Set up redis
+      run: |
+          sudo mkdir -p /usr/share/keyrings
+          curl -fsSL https://packages.redis.io/gpg | sudo -E gpg --yes --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+          echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
+          sudo -E apt-get update
+          sudo -E apt-get install -y redis redis-tools
+          redis-server --daemonize yes
+          redis-cli ping
+
+    - name: Set up mkl
+      run: |
+        cd /tmp
+        curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/faiss-lib.zip
+        sudo unzip -o faiss-lib.zip -d /lib
+        sudo ln -fs /lib/libmkl_core.so /lib/libmkl_core.so.2
+        sudo ln -fs /lib/libmkl_gnu_thread.so /lib/libmkl_gnu_thread.so.2
+        sudo ln -fs /lib/libmkl_intel_lp64.so /lib/libmkl_intel_lp64.so.2
+
+    - name: Prepare data
+      run: |
+        mkdir -p /tmp/friesian-unittest
+        cd /tmp/friesian-unittest
+        curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/friesian-serving-upload.tar.gz 
+        tar -zxf friesian-serving-upload.tar.gz
+
+    - name: Run tests
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        export KERAS_BACKEND=tensorflow
+        cd scala
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_2.x -Dspark.version=2.4.6 "-Dspark.master=local[*]"
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_3.x -Dspark.version=3.1.2 "-Dspark.master=local[*]"
diff --git a/.github/actions/orca-jep-exampletest-action/action.yml b/.github/actions/orca-jep-exampletest-action/action.yml
@@ -0,0 +1,40 @@
+name: 'Run Orca Jep ExampleTest'
+description: 'Run Orca Jep ExampleTest'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Env
+      run: |
+        apt-get update
+        apt-get install wget
+    - name: Set up Python 
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools==58.0.4
+        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
+        pip uninstall -r python/orca/dev/example/requirements-jep.txt -y
+        pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca
+        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-jep.txt
+        pip uninstall -y opencv-python-headless
+        pip install opencv-python-headless
+
+    - name: Run Test
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-tests-jep.sh
+        chmod a+x apps/run-app-tests-pytorch-pip.sh
+        if [ -f /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth ]
+        then
+          rm /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth
+        fi
+        wget -P /root/.cache/torch/hub/checkpoints/ $HTTP_URI/analytics-zoo-models/face-generation/celebaHQ16_december_s7_i96000-9c72988c.pth
+
+        bash python/orca/dev/example/run-example-tests-jep.sh
+        bash apps/run-app-tests-pytorch-pip.sh
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/orca-python-exampletest-action/action.yml b/.github/actions/orca-python-exampletest-action/action.yml
@@ -0,0 +1,35 @@
+name: 'Run Orca Python ExampleTest'
+description: 'Run Orca Python ExampleTest'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Env
+      run: |
+        apt-get update
+        apt-get install wget
+
+    - name: Set up Python 
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7.6'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools==58.0.4
+        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
+        pip uninstall -r python/orca/dev/example/requirements-ray.txt -y
+        pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3
+        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt 
+        pip uninstall -y opencv-python-headless
+        pip install opencv-python-headless
+    - name: Run Test
+      run: |
+        export MAVEN_OPTS="-XX:MaxPermSize=3G -Xmx100G -XX:ReservedCodeCacheSize=512m"
+        export _JAVA_OPTIONS="-Xmx100G -XX:MaxPermSize=3G"
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-tests.sh
+        python/orca/dev/example/run-example-tests.sh
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/orca-python-exampletest-ray-action/action.yml b/.github/actions/orca-python-exampletest-ray-action/action.yml
@@ -0,0 +1,34 @@
+name: 'Run Orca Python ExampleTest Ray'
+description: 'Run Orca Python ExampleTest Ray'
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Env
+      run: |
+        apt-get update
+        apt-get install wget
+
+    - name: Set up Python 
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools==58.0.4
+        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
+        pip uninstall -r python/orca/dev/example/requirements-ray.txt -y
+        pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3
+        pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt 
+        pip uninstall -y opencv-python-headless
+        pip install opencv-python-headless
+
+    - name: Run Test
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        chmod a+x python/orca/dev/example/run-example-test-ray.sh
+        python/orca/dev/example/run-example-test-ray.sh
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}
+        ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
diff --git a/.github/actions/ppml-scala-ut-action/action.yml b/.github/actions/ppml-scala-ut-action/action.yml
@@ -0,0 +1,12 @@
+name: 'Run PPML Scala UT'
+description: 'Run PPML Scala UT'
+runs:
+  using: "composite"
+  steps:
+    - name: Run tests
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        export KERAS_BACKEND=tensorflow
+        cd scala
+        mvn "-DwildcardSuites=com.intel.analytics.bigdl.ppml" "-Dtest=com.intel.analytics.bigdl.ppml.*Test" test -P spark_3.x -Dspark.version=3.1.2 -Ddata-store-url=http://10.239.45.10:8081/repository/raw
+