Add Xshards test to github actions

.github/workflows/orca-tutorial-xshards.yml

@@ -0,0 +1,90 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Orca Tutorial Xshards
+
+on:
+  #release:
+   # types: [published]
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'python/orca/src/bigdl/**'
+      - 'python/orca/tutorial/**'
+      - '.github/workflows/orca-tutorial-xshards.yml'
+  schedule:
+    - cron: '0 15 * * *'
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+
+    runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts]
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up JDK 8
+      uses: actions/setup-java@v3
+      with:
+        java-version: '8'
+        distribution: 'temurin'
+        #server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
+        settings-path: ${{ github.workspace }} # location for the settings.xml file
+
+    - name: Set up Maven
+      uses: stCarolas/[email protected]
+      with:
+        maven-version: 3.8.2
+
+    - name: Set up Maven Settings
+      uses: s4u/[email protected]
+      with:
+        sonatypeSnapshots: true
+        apacheSnapshots: true
+        servers: |
+          [{
+            "id": "central",
+            "configuration": {
+              "httpConfiguration": {
+                "all": {
+                  "connectionTimeout": "3600000",
+                  "readTimeout": "3600000"
+                  }
+                }
+              }
+          }]
+        mirrors: '[{"id": "ardaNexus", "name": "ardaNexus", "mirrorOf": "*", "url": "${NEXUS_URL}" }]'
+
+    - name: Setup Env
+      run: |
+        apt-get update
+        apt-get install wget
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.7'
+    - name: Install dependencies
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install --upgrade setuptools==58.0.4
+        pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
+        pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3
+    - name: Run Test
+      #run: python -m build
+      run: |
+        export SPARK_LOCAL_HOSTNAME=localhost
+        export FTP_URI=ftp://zoo:[email protected]
+        chmod a+x python/orca/dev/test/run-tutorial-xshards.sh
+        python/orca/dev/test/run-tutorial-xshards.sh
+      env:
+        BIGDL_ROOT: ${{ github.workspace }}

python/orca/dev/test/run-tutorial-xshards.sh

@@ -0,0 +1,39 @@
+#
+# Copyright 2016 The BigDL Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+export FTP_URI=$FTP_URI
+set -ex
+
+cd "`dirname $0`"
+
+export PYSPARK_PYTHON=python
+export PYSPARK_DRIVER_PYTHON=python
+
+ray stop -f
+
+cd ../../tutorial/xshards
+echo "Running RayOnSpark tests"
+
+if [ -f ${BIGDL_ROOT}/python/orca/tutorial/xshards/train.csv ]
+then
+    echo "train.csv already exists"
+else
+    wget -nv $FTP_URI/analytics-zoo-data/xshards/train.csv -P ${BIGDL_ROOT}/python/orca/tutorial/xshards/
+fi
+
+#sed -i "s/train.csv/train.csv/g" ${BIGDL_ROOT}/python/orca/tutorial/xshards/tabular_playground_series.py
+
+python tabular_playground_series.py