From 1476e94f616f2a8d8fda212260d2389aa75689c6 Mon Sep 17 00:00:00 2001 From: Shaojun Liu <61072813+liu-shaojun@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:58:34 +0800 Subject: [PATCH] cicd: refactor the github actions workflows (#5529) * add composite actions * add manually build --- .../actions/dllib-scala-ut-action/action.yml | 21 ++ .../friesian-scala-ut-action/action.yml | 38 +++ .../orca-jep-exampletest-action/action.yml | 40 +++ .../orca-python-exampletest-action/action.yml | 35 +++ .../action.yml | 34 +++ .../actions/ppml-scala-ut-action/action.yml | 12 + .github/workflows/manually_build.yml | 254 ++++++++++++++++++ .github/workflows/nightly_build.yml | 188 +------------ .github/workflows/nightly_test.yml | 192 ++----------- 9 files changed, 470 insertions(+), 344 deletions(-) create mode 100644 .github/actions/dllib-scala-ut-action/action.yml create mode 100644 .github/actions/friesian-scala-ut-action/action.yml create mode 100644 .github/actions/orca-jep-exampletest-action/action.yml create mode 100644 .github/actions/orca-python-exampletest-action/action.yml create mode 100644 .github/actions/orca-python-exampletest-ray-action/action.yml create mode 100644 .github/actions/ppml-scala-ut-action/action.yml create mode 100644 .github/workflows/manually_build.yml diff --git a/.github/actions/dllib-scala-ut-action/action.yml b/.github/actions/dllib-scala-ut-action/action.yml new file mode 100644 index 00000000000..d46d3a5cedf --- /dev/null +++ b/.github/actions/dllib-scala-ut-action/action.yml @@ -0,0 +1,21 @@ +name: 'Run Dllib Scala UT' +description: 'Run Dllib Scala UT' +runs: + using: "composite" + steps: + - name: Run tests + run: | + export SPARK_LOCAL_HOSTNAME=localhost + export KERAS_BACKEND=tensorflow + cd scala + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.1.2 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.1.2 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.1.2 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.1.2 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.1.2 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.1.2 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.1.2 + mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.1.2 + #need python requirements + #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" "-Dtest=com.intel.analytics.bigdl.dllib.keras.*Test" test -P spark_3.x -Dspark.version=3.1.2 + #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nn.mkldnn" "-Dtest=com.intel.analytics.bigdl.dllib.nn.mkldnn.*Test" test -P spark_3.x -Dspark.version=3.1.2 diff --git a/.github/actions/friesian-scala-ut-action/action.yml b/.github/actions/friesian-scala-ut-action/action.yml new file mode 100644 index 00000000000..9eb7a7b63f6 --- /dev/null +++ b/.github/actions/friesian-scala-ut-action/action.yml @@ -0,0 +1,38 @@ +name: 'Run Friesian Scala UT' +description: 'Run Friesian Scala UT' +runs: + using: "composite" + steps: + - name: Set up redis + run: | + sudo mkdir -p /usr/share/keyrings + curl -fsSL https://packages.redis.io/gpg | sudo -E gpg --yes --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list + sudo -E apt-get update + sudo -E apt-get install -y redis redis-tools + redis-server --daemonize yes + redis-cli ping + + - name: Set up mkl + run: | + cd /tmp + curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/faiss-lib.zip + sudo unzip -o faiss-lib.zip -d /lib + sudo ln -fs /lib/libmkl_core.so /lib/libmkl_core.so.2 + sudo ln -fs /lib/libmkl_gnu_thread.so /lib/libmkl_gnu_thread.so.2 + sudo ln -fs /lib/libmkl_intel_lp64.so /lib/libmkl_intel_lp64.so.2 + + - name: Prepare data + run: | + mkdir -p /tmp/friesian-unittest + cd /tmp/friesian-unittest + curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/friesian-serving-upload.tar.gz + tar -zxf friesian-serving-upload.tar.gz + + - name: Run tests + run: | + export SPARK_LOCAL_HOSTNAME=localhost + export KERAS_BACKEND=tensorflow + cd scala + mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_2.x -Dspark.version=2.4.6 "-Dspark.master=local[*]" + mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_3.x -Dspark.version=3.1.2 "-Dspark.master=local[*]" diff --git a/.github/actions/orca-jep-exampletest-action/action.yml b/.github/actions/orca-jep-exampletest-action/action.yml new file mode 100644 index 00000000000..ff5c048432d --- /dev/null +++ b/.github/actions/orca-jep-exampletest-action/action.yml @@ -0,0 +1,40 @@ +name: 'Run Orca Jep ExampleTest' +description: 'Run Orca Jep ExampleTest' +runs: + using: "composite" + steps: + - name: Setup Env + run: | + apt-get update + apt-get install wget + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools==58.0.4 + pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 + pip uninstall -r python/orca/dev/example/requirements-jep.txt -y + pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca + pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-jep.txt + pip uninstall -y opencv-python-headless + pip install opencv-python-headless + + - name: Run Test + run: | + export SPARK_LOCAL_HOSTNAME=localhost + chmod a+x python/orca/dev/example/run-example-tests-jep.sh + chmod a+x apps/run-app-tests-pytorch-pip.sh + if [ -f /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth ] + then + rm /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth + fi + wget -P /root/.cache/torch/hub/checkpoints/ $HTTP_URI/analytics-zoo-models/face-generation/celebaHQ16_december_s7_i96000-9c72988c.pth + + bash python/orca/dev/example/run-example-tests-jep.sh + bash apps/run-app-tests-pytorch-pip.sh + env: + BIGDL_ROOT: ${{ github.workspace }} + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} \ No newline at end of file diff --git a/.github/actions/orca-python-exampletest-action/action.yml b/.github/actions/orca-python-exampletest-action/action.yml new file mode 100644 index 00000000000..2aac82453d4 --- /dev/null +++ b/.github/actions/orca-python-exampletest-action/action.yml @@ -0,0 +1,35 @@ +name: 'Run Orca Python ExampleTest' +description: 'Run Orca Python ExampleTest' +runs: + using: "composite" + steps: + - name: Setup Env + run: | + apt-get update + apt-get install wget + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7.6' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools==58.0.4 + pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 + pip uninstall -r python/orca/dev/example/requirements-ray.txt -y + pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3 + pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt + pip uninstall -y opencv-python-headless + pip install opencv-python-headless + - name: Run Test + run: | + export MAVEN_OPTS="-XX:MaxPermSize=3G -Xmx100G -XX:ReservedCodeCacheSize=512m" + export _JAVA_OPTIONS="-Xmx100G -XX:MaxPermSize=3G" + export SPARK_LOCAL_HOSTNAME=localhost + chmod a+x python/orca/dev/example/run-example-tests.sh + python/orca/dev/example/run-example-tests.sh + env: + BIGDL_ROOT: ${{ github.workspace }} + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} \ No newline at end of file diff --git a/.github/actions/orca-python-exampletest-ray-action/action.yml b/.github/actions/orca-python-exampletest-ray-action/action.yml new file mode 100644 index 00000000000..d3ba6910025 --- /dev/null +++ b/.github/actions/orca-python-exampletest-ray-action/action.yml @@ -0,0 +1,34 @@ +name: 'Run Orca Python ExampleTest Ray' +description: 'Run Orca Python ExampleTest Ray' +runs: + using: "composite" + steps: + - name: Setup Env + run: | + apt-get update + apt-get install wget + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --upgrade setuptools==58.0.4 + pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 + pip uninstall -r python/orca/dev/example/requirements-ray.txt -y + pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3 + pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt + pip uninstall -y opencv-python-headless + pip install opencv-python-headless + + - name: Run Test + run: | + export SPARK_LOCAL_HOSTNAME=localhost + chmod a+x python/orca/dev/example/run-example-test-ray.sh + python/orca/dev/example/run-example-test-ray.sh + env: + BIGDL_ROOT: ${{ github.workspace }} + ANALYTICS_ZOO_ROOT: ${{ github.workspace }} \ No newline at end of file diff --git a/.github/actions/ppml-scala-ut-action/action.yml b/.github/actions/ppml-scala-ut-action/action.yml new file mode 100644 index 00000000000..5ff9f447fb9 --- /dev/null +++ b/.github/actions/ppml-scala-ut-action/action.yml @@ -0,0 +1,12 @@ +name: 'Run PPML Scala UT' +description: 'Run PPML Scala UT' +runs: + using: "composite" + steps: + - name: Run tests + run: | + export SPARK_LOCAL_HOSTNAME=localhost + export KERAS_BACKEND=tensorflow + cd scala + mvn "-DwildcardSuites=com.intel.analytics.bigdl.ppml" "-Dtest=com.intel.analytics.bigdl.ppml.*Test" test -P spark_3.x -Dspark.version=3.1.2 -Ddata-store-url=http://10.239.45.10:8081/repository/raw + diff --git a/.github/workflows/manually_build.yml b/.github/workflows/manually_build.yml new file mode 100644 index 00000000000..dc9189dd49b --- /dev/null +++ b/.github/workflows/manually_build.yml @@ -0,0 +1,254 @@ +name: Manually Build + +on: + workflow_dispatch: + inputs: + artifact: + description: 'select which job to run("all" will make all jobs run)' + required: true + default: 'all' + type: choice + options: + - all + - bigdl-ppml-trusted-big-data-ml-python-gramine + - bigdl-ppml-trusted-big-data-ml-python-graphene + - bigdl-ppml-trusted-big-data-ml-scala-occlum + - bigdl-ppml-trusted-realtime-ml-scala-graphene + - bigdl-ppml-trusted-realtime-ml-scala-occlum + - bigdl-ppml-kmsutil + - bigdl-ppml-pccs + tag: + description: 'docker image tag (e.g. 2.1.0-SNAPSHOT)' + required: true + default: 'latest' + type: string + +env: + TAG: ${{ github.event.inputs.tag }} + +permissions: + contents: read + packages: write + +jobs: + + bigdl-ppml-trusted-big-data-ml-python-gramine: + if: ${{ github.event.inputs.artifact == 'bigdl-ppml-trusted-big-data-ml-python-gramine' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Shire] + + steps: + - uses: actions/checkout@v3 + - name: docker login + run: | + docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} + - name: bigdl-ppml-trusted-big-data-ml-python-gramine + run: | + echo "########################################" + echo "####### big-data-ml-python-gramine ####" + echo "########################################" + cd ppml/trusted-big-data-ml/python/docker-gramine + export image=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-gramine + sudo docker build \ + --no-cache=true \ + --build-arg http_proxy=${HTTP_PROXY} \ + --build-arg https_proxy=${HTTPS_PROXY} \ + --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ + --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=${JDK_URL} \ + --build-arg no_proxy=${NO_PROXY} \ + --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ + -t ${image}:${TAG} -f ./Dockerfile . + sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} + sudo docker push 10.239.45.10/arda/${image}:${TAG} + sudo docker rmi -f ${image}:${TAG} + + + bigdl-ppml-trusted-big-data-ml-python-graphene: + if: ${{ github.event.inputs.artifact == 'docker-bigdl-ppml-trusted-big-data-ml-python-graphene' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Shire] + + steps: + - uses: actions/checkout@v3 + - name: docker login + run: | + docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} + - name: bigdl-ppml-trusted-big-data-ml-python-graphene + run: | + echo "########################################" + echo "####### big-data-ml-python-graphene ####" + echo "########################################" + cd ppml/trusted-big-data-ml/python/docker-graphene + export image=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene + sudo docker build \ + --no-cache=true \ + --build-arg http_proxy=${HTTP_PROXY} \ + --build-arg https_proxy=${HTTPS_PROXY} \ + --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ + --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=${JDK_URL} \ + --build-arg no_proxy=${NO_PROXY} \ + --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ + -t ${image}:${TAG} -f ./Dockerfile . + sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} + sudo docker push 10.239.45.10/arda/${image}:${TAG} + sudo docker rmi -f ${image}:${TAG} + + + bigdl-ppml-trusted-big-data-ml-scala-occlum: + if: ${{ github.event.inputs.artifact == 'bigdl-ppml-trusted-big-data-ml-scala-occlum' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Shire] + + steps: + - uses: actions/checkout@v3 + - name: docker login + run: | + docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} + - name: bigdl-ppml-trusted-big-data-ml-scala-occlum + run: | + echo "########################################" + echo "####### big-data-ml-scala-occlum ######" + echo "########################################" + cd ppml/trusted-big-data-ml/scala/docker-occlum/ + export image=intelanalytics/bigdl-ppml-trusted-big-data-ml-scala-occlum + pwd + docker build \ + --no-cache=true \ + --build-arg http_proxy=${HTTP_PROXY} \ + --build-arg https_proxy=${HTTPS_PROXY} \ + --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ + --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=${JDK_URL} \ + --build-arg no_proxy=${NO_PROXY} \ + --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ + -t ${image}:${TAG} -f ./Dockerfile . + docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} + docker push 10.239.45.10/arda/${image}:${TAG} + docker rmi -f ${image}:${TAG} + + + bigdl-ppml-trusted-realtime-ml-scala-graphene: + if: ${{ github.event.inputs.artifact == 'bigdl-ppml-trusted-realtime-ml-scala-graphene' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Shire] + + steps: + - uses: actions/checkout@v3 + - name: docker login + run: | + docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} + - name: bigdl-ppml-trusted-realtime-ml-scala-graphene + run: | + echo "########################################" + echo "####### realtime-ml-scala-graphene #####" + echo "########################################" + cd ppml/trusted-realtime-ml/scala/docker-graphene/ + export image=intelanalytics/bigdl-ppml-trusted-realtime-ml-scala-graphene + pwd + docker build \ + --no-cache=true \ + --build-arg http_proxy=${HTTP_PROXY} \ + --build-arg https_proxy=${HTTPS_PROXY} \ + --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ + --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=${JDK_URL} \ + --build-arg no_proxy=${NO_PROXY} \ + --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ + -t ${image}:${TAG} -f ./Dockerfile . + docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} + docker push 10.239.45.10/arda/${image}:${TAG} + docker rmi -f ${image}:${TAG} + + + bigdl-ppml-trusted-realtime-ml-scala-occlum: + if: ${{ github.event.inputs.artifact == 'bigdl-ppml-trusted-realtime-ml-scala-occlum' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Shire] + + steps: + - uses: actions/checkout@v3 + - name: docker login + run: | + docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} + - name: bigdl-ppml-trusted-realtime-ml-scala-occlum + run: | + echo "########################################" + echo "####### realtime-ml-scala-occlum ######" + echo "########################################" + cd ppml/trusted-realtime-ml/scala/docker-occlum/ + export image=intelanalytics/bigdl-ppml-trusted-realtime-ml-scala-occlum + pwd + docker build \ + --no-cache=true \ + --build-arg http_proxy=${HTTP_PROXY} \ + --build-arg https_proxy=${HTTPS_PROXY} \ + --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ + --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ + --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ + --build-arg JDK_VERSION=8u192 \ + --build-arg JDK_URL=${JDK_URL} \ + --build-arg no_proxy=${NO_PROXY} \ + --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ + -t ${image}:${TAG} -f ./Dockerfile . + docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} + docker push 10.239.45.10/arda/${image}:${TAG} + docker rmi -f ${image}:${TAG} + + + bigdl-ppml-kmsutil: + if: ${{ github.event.inputs.artifact == 'bigdl-ppml-kmsutil' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Shire] + + steps: + - uses: actions/checkout@v3 + - name: docker deploy kms-util + run: | + export IMAGE=intelanalytics/kms-utils + cd ppml/services/kms-utils/docker + echo "########################################" + echo "############## kms-utils ###############" + echo "########################################" + docker build \ + --no-cache=true \ + --build-arg http_proxy=${HTTP_PROXY} \ + --build-arg https_proxy=${HTTPS_PROXY} \ + --build-arg JDK_URL=${JDK_URL} \ + -t $IMAGE:$TAG -f ./Dockerfile . + docker tag ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}:${TAG} + docker push 10.239.45.10/arda/${IMAGE}:${TAG} + docker rmi -f ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}:${TAG} + + + bigdl-ppml-pccs: + if: ${{ github.event.inputs.artifact == 'bigdl-ppml-pccs' || github.event.inputs.artifact == 'all' }} + runs-on: [self-hosted, Shire] + + steps: + - uses: actions/checkout@v3 + - name: docker deploy pccs + run: | + export IMAGE=intelanalytics/pccs + cd ppml/services/pccs/docker + echo "########################################" + echo "################# PCCS #################" + echo "########################################" + docker build \ + --no-cache=true \ + --build-arg http_proxy=${HTTP_PROXY} \ + --build-arg https_proxy=${HTTPS_PROXY} \ + -t $IMAGE:$TAG -f ./Dockerfile . + docker tag ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}:${TAG} + docker push 10.239.45.10/arda/${IMAGE}:${TAG} + docker rmi -f ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}:${TAG} + + diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index 9cbbba0a94b..4f685d0f5c4 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -2,7 +2,7 @@ name: Nightly Build on: schedule: - - cron: '0 17 * * *' + - cron: '0 12 * * *' # GMT time, 12:00 GMT == 20:00 China workflow_dispatch: inputs: artifact: @@ -12,31 +12,15 @@ on: type: choice options: - all - - scala - - python - - python-sourceforge - - docker-bigdl - - docker-bigdl-ppml-trusted-big-data-ml-python-gramine - - docker-bigdl-ppml-trusted-big-data-ml-python-graphene - - docker-bigdl-ppml-trusted-realtime-ml-scala-graphene - - docker-bigdl-ppml-trusted-big-data-ml-scala-occlum - - docker-bigdl-ppml-trusted-realtime-ml-scala-occlum - - docker-kms - bigdlTag: - description: 'e.g. 2.1.0-SNAPSHOT' + - scala-build + - python-build + - python-sourceforge-build + - docker-bigdl-build + tag: + description: 'docker image tag (e.g. 2.1.0-SNAPSHOT)' required: true default: 'latest' type: string - bigdlPpmlTag: - description: 'e.g. 2.1.0-SNAPSHOT' - required: true - default: 'latest' - type: string - kmsTag: - description: 'e.g. 0.3.0' - required: true - default: '0.3.0' - type: string permissions: contents: read @@ -45,7 +29,7 @@ permissions: jobs: scala-build: - if: ${{ github.event.inputs.artifact == 'scala' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'scala-build' || github.event.inputs.artifact == 'docker-bigdl-build' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Linux, Bree] steps: @@ -169,7 +153,7 @@ jobs: docker-build-bigdl: needs: scala-build - if: ${{ github.event.inputs.artifact == 'bigdl' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'docker-bigdl-build' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Shire] steps: @@ -221,155 +205,9 @@ jobs: docker push 10.239.45.10/arda/${IMAGE}-spark-3.1.2:${TAG} docker rmi -f ${IMAGE}-spark-3.1.2:${TAG} - docker-build-ppml: - needs: scala-build - if: ${{ !github.event.schedule && (github.event.inputs.artifact == 'docker-bigdl-ppml-trusted-big-data-ml-python-gramine' || github.event.inputs.artifact == 'docker-bigdl-ppml-trusted-big-data-ml-python-graphene' || github.event.inputs.artifact == 'docker-bigdl-ppml-trusted-realtime-ml-scala-graphene' || github.event.inputs.artifact == 'docker-bigdl-ppml-trusted-big-data-ml-scala-occlum' || github.event.inputs.artifact == 'docker-bigdl-ppml-trusted-realtime-ml-scala-occlum' || github.event.inputs.artifact == 'all') }} - runs-on: [self-hosted, Shire] - steps: - - uses: actions/checkout@v3 - - name: docker login - run: | - docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} - - name: Set the variable - env: - DEFAULT_TAG: 'latest' - DEFAULT_ARTIFACT: 'all' - run: | - echo "TAG=${{ github.event.inputs.bigdlPpmlTag || env.DEFAULT_TAG }} " >> $GITHUB_ENV - echo "ARTIFACT=${{ github.event.inputs.artifact || env.DEFAULT_ARTIFACT }}" >> $GITHUB_ENV - - name: bigdl-ppml-trusted-big-data-ml-python-gramine - run: | - if [ "$ARTIFACT" = "docker-bigdl-ppml-trusted-big-data-ml-python-gramine" ] || [ "$ARTIFACT" = "all" ] ; then - echo "########################################" - echo "####### big-data-ml-python-gramine ####" - echo "########################################" - cd ppml/trusted-big-data-ml/python/docker-gramine - export image=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-gramine - sudo docker build \ - --no-cache=true \ - --build-arg http_proxy=${HTTP_PROXY} \ - --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ - --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ - --build-arg JDK_VERSION=8u192 \ - --build-arg JDK_URL=${JDK_URL} \ - --build-arg no_proxy=${NO_PROXY} \ - --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ - -t ${image}:${TAG} -f ./Dockerfile . - sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} - sudo docker push 10.239.45.10/arda/${image}:${TAG} - sudo docker rmi -f ${image}:${TAG} - fi - - name: bigdl-ppml-trusted-big-data-ml-python-graphene - run: | - if [[ "$ARTIFACT" == docker-bigdl-ppml-trusted-big-data-ml-python-graphene || "$ARTIFACT" == all ]]; then - echo "########################################" - echo "####### big-data-ml-python-graphene ####" - echo "########################################" - cd ppml/trusted-big-data-ml/python/docker-graphene - export image=intelanalytics/bigdl-ppml-trusted-big-data-ml-python-graphene - sudo docker build \ - --no-cache=true \ - --build-arg http_proxy=${HTTP_PROXY} \ - --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ - --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ - --build-arg JDK_VERSION=8u192 \ - --build-arg JDK_URL=${JDK_URL} \ - --build-arg no_proxy=${NO_PROXY} \ - --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ - -t ${image}:${TAG} -f ./Dockerfile . - sudo docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} - sudo docker push 10.239.45.10/arda/${image}:${TAG} - sudo docker rmi -f ${image}:${TAG} - fi - - name: bigdl-ppml-trusted-realtime-ml-scala-graphene - run: | - if [[ "$ARTIFACT" == docker-bigdl-ppml-trusted-realtime-ml-scala-graphene || "$ARTIFACT" == all ]]; then - echo "########################################" - echo "####### realtime-ml-scala-graphene #####" - echo "########################################" - cd ppml/trusted-realtime-ml/scala/docker-graphene/ - export image=intelanalytics/bigdl-ppml-trusted-realtime-ml-scala-graphene - pwd - docker build \ - --no-cache=true \ - --build-arg http_proxy=${HTTP_PROXY} \ - --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ - --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ - --build-arg JDK_VERSION=8u192 \ - --build-arg JDK_URL=${JDK_URL} \ - --build-arg no_proxy=${NO_PROXY} \ - --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ - -t ${image}:${TAG} -f ./Dockerfile . - docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} - docker push 10.239.45.10/arda/${image}:${TAG} - docker rmi -f ${image}:${TAG} - fi - - name: bigdl-ppml-trusted-big-data-ml-scala-occlum - run: | - if [[ "$ARTIFACT" == docker-bigdl-ppml-trusted-big-data-ml-scala-occlum || "$ARTIFACT" == all ]]; then - echo "########################################" - echo "####### big-data-ml-scala-occlum ######" - echo "########################################" - cd ppml/trusted-big-data-ml/scala/docker-occlum/ - export image=intelanalytics/bigdl-ppml-trusted-big-data-ml-scala-occlum - pwd - docker build \ - --no-cache=true \ - --build-arg http_proxy=${HTTP_PROXY} \ - --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ - --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ - --build-arg JDK_VERSION=8u192 \ - --build-arg JDK_URL=${JDK_URL} \ - --build-arg no_proxy=${NO_PROXY} \ - --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ - -t ${image}:${TAG} -f ./Dockerfile . - docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} - docker push 10.239.45.10/arda/${image}:${TAG} - docker rmi -f ${image}:${TAG} - fi - - name: bigdl-ppml-trusted-realtime-ml-scala-occlum - run: | - if [[ "$ARTIFACT" == docker-bigdl-ppml-trusted-realtime-ml-scala-occlum || "$ARTIFACT" == all ]]; then - echo "########################################" - echo "####### realtime-ml-scala-occlum ######" - echo "########################################" - cd ppml/trusted-realtime-ml/scala/docker-occlum/ - export image=intelanalytics/bigdl-ppml-trusted-realtime-ml-scala-occlum - pwd - docker build \ - --no-cache=true \ - --build-arg http_proxy=${HTTP_PROXY} \ - --build-arg https_proxy=${HTTPS_PROXY} \ - --build-arg HTTP_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTP_PROXY_PORT=${HTTP_PROXY_PORT_2} \ - --build-arg HTTPS_PROXY_HOST=${HTTP_PROXY_HOST_2} \ - --build-arg HTTPS_PROXY_PORT=${HTTP_PROXY_PORT_3} \ - --build-arg JDK_VERSION=8u192 \ - --build-arg JDK_URL=${JDK_URL} \ - --build-arg no_proxy=${NO_PROXY} \ - --build-arg SPARK_JAR_REPO_URL=${SPARK_JAR_REPO_URL} \ - -t ${image}:${TAG} -f ./Dockerfile . - docker tag ${image}:${TAG} 10.239.45.10/arda/${image}:${TAG} - docker push 10.239.45.10/arda/${image}:${TAG} - docker rmi -f ${image}:${TAG} - fi - - docker-build-kms: needs: scala-build - if: ${{ github.event.schedule == '0 17 * * 1' || github.event.inputs.artifact == 'docker-kms' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule == '0 12 * * 1' || github.event.inputs.artifact == 'docker-kms' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Shire] steps: @@ -411,7 +249,7 @@ jobs: docker rmi -f ${IMAGE}:${TAG} 10.239.45.10/arda/${IMAGE}:${TAG} python-build: - if: ${{ github.event.inputs.artifact == 'python' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'python-build' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, ubuntu-20.04-lts, Bree] steps: @@ -449,8 +287,8 @@ jobs: ## mac ## bash python/dev/release_default_mac.sh ${nb_version} true - python-build-sourceforge: - if: ${{ github.event.inputs.artifact == 'python-sourceforge' || github.event.inputs.artifact == 'all' }} + python-sourceforge-build: + if: ${{ github.event.schedule || github.event.inputs.artifact == 'python-sourceforge-build' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted,alpha,Bree,ubuntu-20.04-lts] steps: diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml index 787500d8107..8ddee2da9ba 100644 --- a/.github/workflows/nightly_test.yml +++ b/.github/workflows/nightly_test.yml @@ -2,7 +2,7 @@ name: Nightly Test on: schedule: - - cron: '0 15 * * *' + - cron: '0 13 * * *' # GMT time, 13:00 GMT == 21:00 China workflow_dispatch: inputs: artifact: @@ -26,93 +26,34 @@ permissions: jobs: Orca-Python-ExampleTest-Py37-Spark3: - if: ${{ github.event.inputs.artifact == 'Orca-Python-ExampleTest-Py37-Spark3' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-ExampleTest-Py37-Spark3' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] steps: - uses: actions/checkout@v3 - name: Set up JDK8 uses: ./.github/actions/jdk-setup-action - - name: Set up maven uses: ./.github/actions/maven-setup-action + - name: Run test + uses: ./.github/actions/orca-python-exampletest-action - - name: Setup Env - run: | - apt-get update - apt-get install wget - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7.6' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools==58.0.4 - pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 - pip uninstall -r python/orca/dev/example/requirements-ray.txt -y - pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3 - pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt - pip uninstall -y opencv-python-headless - pip install opencv-python-headless - - name: Run Test - run: | - export MAVEN_OPTS="-XX:MaxPermSize=3G -Xmx100G -XX:ReservedCodeCacheSize=512m" - export _JAVA_OPTIONS="-Xmx100G -XX:MaxPermSize=3G" - export SPARK_LOCAL_HOSTNAME=localhost - chmod a+x python/orca/dev/example/run-example-tests.sh - python/orca/dev/example/run-example-tests.sh - env: - BIGDL_ROOT: ${{ github.workspace }} - ANALYTICS_ZOO_ROOT: ${{ github.workspace }} Orca-Python-ExampleTest-Ray-Py37-Spark3: - if: ${{ github.event.inputs.artifact == 'Orca-Python-ExampleTest-Ray-Py37-Spark3' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Python-ExampleTest-Ray-Py37-Spark3' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] steps: - uses: actions/checkout@v3 - - name: Set up JDK8 uses: ./.github/actions/jdk-setup-action - - name: Set up maven uses: ./.github/actions/maven-setup-action - - - name: Setup Env - run: | - apt-get update - apt-get install wget - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools==58.0.4 - pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 - pip uninstall -r python/orca/dev/example/requirements-ray.txt -y - pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca-spark3 - pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-ray.txt - pip uninstall -y opencv-python-headless - pip install opencv-python-headless - - - name: Run Test - run: | - export SPARK_LOCAL_HOSTNAME=localhost - chmod a+x python/orca/dev/example/run-example-test-ray.sh - python/orca/dev/example/run-example-test-ray.sh - env: - BIGDL_ROOT: ${{ github.workspace }} - ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + - name: Run test + uses: ./.github/actions/orca-python-exampletest-ray-action Orca-Jep-ExampleTest-Py37-Spark2: - if: ${{ github.event.inputs.artifact == 'Orca-Jep-ExampleTest-Py37-Spark2' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Orca-Jep-ExampleTest-Py37-Spark2' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] steps: @@ -121,45 +62,11 @@ jobs: uses: ./.github/actions/jdk-setup-action - name: Set up maven uses: ./.github/actions/maven-setup-action - - - name: Setup Env - run: | - apt-get update - apt-get install wget - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.7' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install --upgrade setuptools==58.0.4 - pip uninstall -y bigdl-friesian bigdl-friesian-spark3 bigdl-dllib bigdl-dllib-spark3 bigdl-orca pyspark bigdl-orca-spark3 bigdl-chronos bigdl-chronos-spark3 bigdl-friesian bigdl-friesian-spark3 - pip uninstall -r python/orca/dev/example/requirements-jep.txt -y - pip install -i https://pypi.org/simple --pre --upgrade bigdl-orca - pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r python/orca/dev/example/requirements-jep.txt - pip uninstall -y opencv-python-headless - pip install opencv-python-headless - - - name: Run Test - run: | - export SPARK_LOCAL_HOSTNAME=localhost - chmod a+x python/orca/dev/example/run-example-tests-jep.sh - chmod a+x apps/run-app-tests-pytorch-pip.sh - if [ -f /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth ] - then - rm /root/.cache/torch/hub/checkpoints/celebaHQ16_december_s7_i96000-9c72988c.pth - fi - wget -P /root/.cache/torch/hub/checkpoints/ $HTTP_URI/analytics-zoo-models/face-generation/celebaHQ16_december_s7_i96000-9c72988c.pth - - bash python/orca/dev/example/run-example-tests-jep.sh - bash apps/run-app-tests-pytorch-pip.sh - env: - BIGDL_ROOT: ${{ github.workspace }} - ANALYTICS_ZOO_ROOT: ${{ github.workspace }} + - name: Run test + uses: ./.github/actions/orca-jep-exampletest-action Scala-App: - if: ${{ github.event.inputs.artifact == 'Scala-App' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Scala-App' || github.event.inputs.artifact == 'all' }} runs-on: [self-hosted, Gondolin, ubuntu-20.04-lts] steps: @@ -168,7 +75,6 @@ jobs: uses: ./.github/actions/jdk-setup-action - name: Set up maven uses: ./.github/actions/maven-setup-action - - name: Set up Python uses: actions/setup-python@v2 with: @@ -194,7 +100,7 @@ jobs: apps/run-scala-app-test.sh Dllib-Scala-UT: - if: ${{ github.event.inputs.artifact == 'Dllib-Scala-UT' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Dllib-Scala-UT' || github.event.inputs.artifact == 'all' }} runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ] steps: @@ -203,26 +109,11 @@ jobs: uses: ./.github/actions/jdk-setup-action - name: Set up maven uses: ./.github/actions/maven-setup-action - - - name: Run tests - run: | - export SPARK_LOCAL_HOSTNAME=localhost - export KERAS_BACKEND=tensorflow - cd scala - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.optim" "-Dtest=com.intel.analytics.bigdl.dllib.optim.*Test" test -P spark_3.x -Dspark.version=3.1.2 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.models" "-Dtest=com.intel.analytics.bigdl.dllib.models.*Test" test -P spark_3.x -Dspark.version=3.1.2 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.estimator" "-Dtest=com.intel.analytics.bigdl.dllib.estimator.*Test" test -P spark_3.x -Dspark.version=3.1.2 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nnframes" "-Dtest=com.intel.analytics.bigdl.dllib.nnframes.*Test" test -P spark_3.x -Dspark.version=3.1.2 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.feature" "-Dtest=com.intel.analytics.bigdl.dllib.feature.*Test" test -P spark_3.x -Dspark.version=3.1.2 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.intermediate" "-Dtest=com.intel.analytics.bigdl.dllib.utils.intermediate.*Test" test -P spark_3.x -Dspark.version=3.1.2 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.tf" "-Dtest=com.intel.analytics.bigdl.dllib.utils.tf.*Test" test -P spark_3.x -Dspark.version=3.1.2 - mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.utils.python.api" "-Dtest=com.intel.analytics.bigdl.dllib.utils.python.api.*Test" test -P spark_3.x -Dspark.version=3.1.2 - #need python requirements - #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.keras" "-Dtest=com.intel.analytics.bigdl.dllib.keras.*Test" test -P spark_3.x -Dspark.version=3.1.2 - #mvn "-DwildcardSuites=com.intel.analytics.bigdl.dllib.nn.mkldnn" "-Dtest=com.intel.analytics.bigdl.dllib.nn.mkldnn.*Test" test -P spark_3.x -Dspark.version=3.1.2 - + - name: Run test + uses: ./.github/actions/dllib-scala-ut-action + Friesian-Scala-UT: - if: ${{ github.event.inputs.artifact == 'Friesian-Scala-UT' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'Friesian-Scala-UT' || github.event.inputs.artifact == 'all' }} runs-on: [ self-hosted, ubuntu-20.04-lts, CLX, AVX512, Ettenmoors ] steps: @@ -231,43 +122,11 @@ jobs: uses: ./.github/actions/jdk-setup-action - name: Set up maven uses: ./.github/actions/maven-setup-action - - - name: Set up redis - run: | - sudo mkdir -p /usr/share/keyrings - curl -fsSL https://packages.redis.io/gpg | sudo -E gpg --yes --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg - echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list - sudo -E apt-get update - sudo -E apt-get install -y redis redis-tools - redis-server --daemonize yes - redis-cli ping - - - name: Set up mkl - run: | - cd /tmp - curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/faiss-lib.zip - sudo unzip -o faiss-lib.zip -d /lib - sudo ln -fs /lib/libmkl_core.so /lib/libmkl_core.so.2 - sudo ln -fs /lib/libmkl_gnu_thread.so /lib/libmkl_gnu_thread.so.2 - sudo ln -fs /lib/libmkl_intel_lp64.so /lib/libmkl_intel_lp64.so.2 - - - name: Prepare data - run: | - mkdir -p /tmp/friesian-unittest - cd /tmp/friesian-unittest - curl -O http://10.239.45.10:8081/repository/raw/analytics-zoo-data/friesian-serving-upload.tar.gz - tar -zxf friesian-serving-upload.tar.gz - - - name: Run tests - run: | - export SPARK_LOCAL_HOSTNAME=localhost - export KERAS_BACKEND=tensorflow - cd scala - mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_2.x -Dspark.version=2.4.6 "-Dspark.master=local[*]" - mvn "-DwildcardSuites=com.intel.analytics.bigdl.friesian" "-Dtest=com.intel.analytics.bigdl.friesian.*Test" test -P spark_3.x -Dspark.version=3.1.2 "-Dspark.master=local[*]" - + - name: Run test + uses: ./.github/actions/friesian-scala-ut-action + PPML-Scala-UT: - if: ${{ github.event.inputs.artifact == 'PPML-Scala-UT' || github.event.inputs.artifact == 'all' }} + if: ${{ github.event.schedule || github.event.inputs.artifact == 'PPML-Scala-UT' || github.event.inputs.artifact == 'all' }} runs-on: [ self-hosted, Gondolin, ubuntu-20.04-lts ] steps: @@ -276,14 +135,9 @@ jobs: uses: ./.github/actions/jdk-setup-action - name: Set up maven uses: ./.github/actions/maven-setup-action - - name: Run tests - run: | - export SPARK_LOCAL_HOSTNAME=localhost - export KERAS_BACKEND=tensorflow - cd scala - mvn "-DwildcardSuites=com.intel.analytics.bigdl.ppml" "-Dtest=com.intel.analytics.bigdl.ppml.*Test" test -P spark_3.x -Dspark.version=3.1.2 -Ddata-store-url=http://10.239.45.10:8081/repository/raw - - + - name: Run test + uses: ./.github/actions/ppml-scala-ut-action +