Skip to content

Commit

Permalink
Orca: Add orca ray ctx example to run on self hosted (#5484)
Browse files Browse the repository at this point in the history
* Add orca ray ctx example in github workflows

* Install cmake 3.16.3 with pip

* Increase the CPU limits

* Add newline at EOF

* Align with the new format

* Add the pull_request trigger

* Add missing property

* Uninstall spark first

* Uninstall spark3

* Remove test triggers

* Add tests to PR_validation

* Fix a bug

* Test the validation trigger

* Undo the change
  • Loading branch information
aixideng authored Aug 29, 2022
1 parent 91c482f commit 8f6a758
Show file tree
Hide file tree
Showing 3 changed files with 79 additions and 15 deletions.
43 changes: 43 additions & 0 deletions .github/actions/orca-ray-ctx-example-action/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: 'Run Orca Ray Ctx Example'
description: 'Run Orca Ray Ctx Example'
runs:
using: "composite"
steps:
- name: Setup Env
shell: bash
run: |
apt-get update
apt-get install wget
- name: Set up Python
uses: actions/setup-python@v2
with:
python-version: '3.7.11'

- name: Install dependencies
shell: bash
run: |
python -m pip install --upgrade pip
python -m pip install --upgrade setuptools==58.0.4
pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3
pip uninstall -r python/orca/dev/example/requirements-ray.txt -y
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt
pip install pytest
pip install cmake==3.16.3
pip install tensorflow==2.3.0
HOROVOD_WITH_TENSORFLOW=1; HOROVOD_WITH_GLOO=1; pip install --no-cache-dir horovod==0.19.2
pip install ray==1.11.0
pip install --pre --upgrade bigdl-orca
- name: Run Test
shell: bash
run: |
export SPARK_LOCAL_HOSTNAME=localhost
chmod a+x python/orca/dev/example/run-example-test-ray-runtime.sh
python/orca/dev/example/run-example-test-ray-runtime.sh
env:
BIGDL_ROOT: ${{ github.workspace }}
ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
25 changes: 24 additions & 1 deletion .github/workflows/PR_validation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,17 @@ on:
- '.github/actions/friesian-scala-ut-action/action.yml'
- 'scala/dllib/**'
- '.github/actions/dllib-scala-ut-action/action.yml'
- 'python/orca/**'
- '.github/actions/orca-ray-ctx-example-action/action.yml'
pull_request:
branches: [ main ]
paths:
- 'scala/friesian/**'
- '.github/actions/friesian-scala-ut-action/action.yml'
- 'scala/dllib/**'
- '.github/actions/dllib-scala-ut-action/action.yml'
- 'python/orca/**'
- '.github/actions/orca-ray-ctx-example-action/action.yml'

permissions:
contents: read
Expand All @@ -26,6 +30,7 @@ jobs:
outputs:
friesian: ${{ steps.filter.outputs.friesian }}
dllib: ${{ steps.filter.outputs.dllib }}
orca: ${{ steps.filter.outputs.orca }}
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
Expand All @@ -38,6 +43,9 @@ jobs:
dllib:
- 'scala/dllib/**'
- '.github/actions/dllib-scala-ut-action/action.yml'
orca:
- 'python/orca/**'
- '.github/actions/orca-ray-ctx-example-action/action.yml'
Friesian-Scala-UT:
Expand Down Expand Up @@ -67,4 +75,19 @@ jobs:
- name: Set up maven
uses: ./.github/actions/maven-setup-action
- name: Run test
uses: ./.github/actions/dllib-scala-ut-action
uses: ./.github/actions/dllib-scala-ut-action


Orca-Ray-Ctx-Example:
needs: changes
if: ${{ needs.changes.outputs.orca == 'true' }}
runs-on: [self-hosted, Gondolin-resources, ubuntu-20.04-lts]

steps:
- uses: actions/checkout@v3
- name: Set up JDK8
uses: ./.github/actions/jdk-setup-action
- name: Set up maven
uses: ./.github/actions/maven-setup-action
- name: Run test
uses: ./.github/actions/orca-ray-ctx-example-action
26 changes: 12 additions & 14 deletions .github/workflows/nightly_test.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
name: Nightly Test

on:
#pull_request:
schedule:
- cron: '0 13 * * *' # GMT time, 13:00 GMT == 21:00 China
workflow_dispatch:
Expand All @@ -21,6 +20,7 @@ on:
- Dllib-Scala-UT
- Friesian-Scala-UT
- PPML-Scala-UT
- Orca-Ray-Ctx-Example

permissions:
contents: read
Expand Down Expand Up @@ -152,18 +152,16 @@ jobs:
uses: ./.github/actions/maven-setup-action
- name: Run test
uses: ./.github/actions/ppml-scala-ut-action














Orca-Ray-Ctx-Example:
if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Ray-Ctx-Example' || github.event.inputs.artifact == 'all' }}
runs-on: [self-hosted, Gondolin-resources, ubuntu-20.04-lts]

steps:
- uses: actions/checkout@v3
- name: Set up JDK8
uses: ./.github/actions/jdk-setup-action
- name: Set up maven
uses: ./.github/actions/maven-setup-action
- name: Run test
uses: ./.github/actions/orca-ray-ctx-example-action

0 comments on commit 8f6a758

Please sign in to comment.