From abacaf4466c1ff0d7167dfe3b274dc21df779c69 Mon Sep 17 00:00:00 2001 From: pinggao187 Date: Tue, 30 Aug 2022 11:08:15 +0800 Subject: [PATCH 1/5] add orca py37 UT to github action --- .../orca-python-py37-spark3-action/action.yml | 37 +++++++++++++++++++ .github/workflows/nightly_test.yml | 17 ++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 .github/actions/orca-python-py37-spark3-action/action.yml diff --git a/.github/actions/orca-python-py37-spark3-action/action.yml b/.github/actions/orca-python-py37-spark3-action/action.yml new file mode 100644 index 00000000000..2cbc1d433e7 --- /dev/null +++ b/.github/actions/orca-python-py37-spark3-action/action.yml @@ -0,0 +1,37 @@ +name: 'Run Orca Python Py37 Spark3' +description: 'Run Orca Python Py37 Spark3' +runs: + using: "composite" + steps: + - name: Setup Env + shell: bash + run: | + apt-get update + apt-get install wget + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools==58.0.4 + pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 + pip uninstall -r python/orca/dev/example/requirements-ray.txt -y + pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3 + pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt + pip install pytest + pip uninstall -y opencv-python-headless + pip install opencv-python-headless + - name: Run Test + shell: bash + run: | + export SPARK_LOCAL_HOSTNAME=localhost + chmod a+x python/orca/dev/test/run-pytests-spark.sh + python/orca/dev/test/run-pytests-spark.sh + env: + BIGDL_ROOT: ${{ github.workspace }} + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index c5b0a5618fc..be6aa19c873 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -16,11 +16,13 @@ on: - Orca-Python-ExampleTest-Ray-Py37-Spark3 - Orca-Jep-ExampleTest-Py37-Spark2 - Orca-Python-Ray-Py37-Spark3 + - Orca-Python-Py37-Spark3 + - Orca-Ray-Ctx-Example - Scala-App - Dllib-Scala-UT - Friesian-Scala-UT - PPML-Scala-UT - - Orca-Ray-Ctx-Example + permissions: contents: read @@ -80,6 +82,19 @@ jobs: - name: Run test uses: ./.github/actions/orca-python-ray-py37-spark3-action + Orca-Python-Py37-Spark3: + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-Py37-Spark3' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK8 + uses: ./.github/actions/jdk-setup-action + - name: Set up maven + uses: ./.github/actions/maven-setup-action + - name: Run test + uses: ./.github/actions/orca-python-py37-spark3-action + Scala-App: if: ${{ github.event.schedule || github.event.inputs.artifact == 'Scala-App' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] From f0e163fd576e79d53a31299ba0952b45b28f4dd4 Mon Sep 17 00:00:00 2001 From: pinggao187 Date: Tue, 30 Aug 2022 11:12:04 +0800 Subject: [PATCH 2/5] update pull_request path --- .github/workflows/nightly_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index be6aa19c873..c30a7005bf3 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -1,6 +1,7 @@ name: Nightly Test on: + pull_request: schedule: - cron: '0 13 * * *' # GMT time, 13:00 GMT == 21:00 China workflow_dispatch: @@ -83,7 +84,7 @@ jobs: uses: ./.github/actions/orca-python-ray-py37-spark3-action Orca-Python-Py37-Spark3: - if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-Py37-Spark3' || github.event.inputs.artifact == 'all' }} + #if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-Py37-Spark3' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] steps: From 97ab5bb28c1177df3a0018430407acbeb344c926 Mon Sep 17 00:00:00 2001 From: pinggao187 Date: Wed, 31 Aug 2022 10:42:17 +0800 Subject: [PATCH 3/5] update build python whl --- .../actions/orca-python-py37-spark3-action/action.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/actions/orca-python-py37-spark3-action/action.yml b/.github/actions/orca-python-py37-spark3-action/action.yml index 2cbc1d433e7..a0b8c3e109e 100644 --- a/.github/actions/orca-python-py37-spark3-action/action.yml +++ b/.github/actions/orca-python-py37-spark3-action/action.yml @@ -21,15 +21,23 @@ runs: python -m pip install --upgrade setuptools==58.0.4 pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 pip uninstall -r python/orca/dev/example/requirements-ray.txt -y - pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3 pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt pip install pytest pip uninstall -y opencv-python-headless pip install opencv-python-headless + bash python/dev/release_default_linux_spark312.sh default false false false -Ddata-store-url=$HTTP_URI -U + + # install dllib + pip install python/dllib/src/dist/bigdl_dllib*-py3-none-manylinux1_x86_64.whl + + # install orca + pip install python/orca/src/dist/bigdl_orca*-py3-none-manylinux1_x86_64.whl + - name: Run Test shell: bash run: | export SPARK_LOCAL_HOSTNAME=localhost + chmod a+x python/orca/dev/test/run-pytests-spark.sh python/orca/dev/test/run-pytests-spark.sh env: From 30914876fd712a6f3c8c9beb307946c1ac1131d9 Mon Sep 17 00:00:00 2001 From: pinggao187 Date: Wed, 31 Aug 2022 11:03:33 +0800 Subject: [PATCH 4/5] update bigdl-core version --- .github/actions/orca-python-py37-spark3-action/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/orca-python-py37-spark3-action/action.yml b/.github/actions/orca-python-py37-spark3-action/action.yml index a0b8c3e109e..943ac0e37bb 100644 --- a/.github/actions/orca-python-py37-spark3-action/action.yml +++ b/.github/actions/orca-python-py37-spark3-action/action.yml @@ -25,6 +25,7 @@ runs: pip install pytest pip uninstall -y opencv-python-headless pip install opencv-python-headless + sed -i "s/'bigdl-core=='+VERSION/'bigdl-core==2.1.0b20220811'/g" python/dllib/src/setup.py bash python/dev/release_default_linux_spark312.sh default false false false -Ddata-store-url=$HTTP_URI -U # install dllib From 145d17d9bcc4178c4858caa165c90d72d69242cf Mon Sep 17 00:00:00 2001 From: pinggao187 Date: Wed, 31 Aug 2022 13:15:56 +0800 Subject: [PATCH 5/5] remove pull_request --- .github/workflows/nightly_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index c30a7005bf3..bcc4d26dd03 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -1,7 +1,7 @@ name: Nightly Test on: - pull_request: + #pull_request: schedule: - cron: '0 13 * * *' # GMT time, 13:00 GMT == 21:00 China workflow_dispatch: @@ -84,7 +84,7 @@ jobs: uses: ./.github/actions/orca-python-ray-py37-spark3-action Orca-Python-Py37-Spark3: - #if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-Py37-Spark3' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-Py37-Spark3' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] steps: