diff --git a/.github/workflows/macos_building.yml b/.github/workflows/macos_building.yml index 561c94cc..95e0c36c 100644 --- a/.github/workflows/macos_building.yml +++ b/.github/workflows/macos_building.yml @@ -7,7 +7,7 @@ on: branches: [ master ] jobs: - streaming-cpp-test: + streaming-mac-pipeline: timeout-minutes: 60 runs-on: ${{ matrix.os }} strategy: @@ -29,5 +29,18 @@ jobs: fi sudo sh scripts/install-bazel.sh - - name: Build streaming and test + #- name: Install ray wheel + # run: | + # which python3 + # python3 --version + # which python3.8 + # python3.8 -m pip3 install https://ray-mobius-us.oss-us-west-1.aliyuncs.com/ci/macos/ray-2.0.0.dev0-cp38-cp38-macosx_11_0_x86_64.whl + + #- name: Streaming python test + # run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_python" + + - name: Streaming java test + run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_java" + + - name: Streaming cpp test run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_cpp" diff --git a/.github/workflows/ubuntu_building.yml b/.github/workflows/ubuntu_building.yml index 2da68386..38b835bb 100644 --- a/.github/workflows/ubuntu_building.yml +++ b/.github/workflows/ubuntu_building.yml @@ -7,7 +7,7 @@ on: branches: [ master ] jobs: - streaming-cpp-test: + streaming-ubuntu-pipeline: timeout-minutes: 60 runs-on: ubuntu-latest container: docker.io/library/ubuntu:latest @@ -18,9 +18,20 @@ jobs: - name: Install bazel run: | apt-get update - apt-get install -yq wget gcc g++ openjdk-8-jdk python3.7 zlib1g-dev zip + apt-get install -yq wget gcc g++ openjdk-8-jdk python3.8 zlib1g-dev zip + apt-get install -yq pip sh scripts/install-bazel.sh + - name: Install ray wheel + run: | + python3 -m pip install virtualenv + python3 -m virtualenv -p python3 py3 + . py3/bin/activate + pip install pytest + pip install https://ray-mobius-us.oss-us-west-1.aliyuncs.com/ci/linux/ubuntu/ray-2.0.0.dev0-cp38-cp38-linux_x86_64.whl + - name: Streaming python test + run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_python" + - name: Streaming java test run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_java" diff --git a/streaming/buildtest.sh b/streaming/buildtest.sh index 07f72f1b..24646057 100755 --- a/streaming/buildtest.sh +++ b/streaming/buildtest.sh @@ -1,6 +1,28 @@ #!/bin/bash script_dir=$(cd "$(dirname "${BASH_SOURCE:-$0}")" || exit; pwd) +function create_py_env() +{ + PY3_DIR=${1} + python3 -m pip install virtualenv + python3 -m virtualenv -p python3 $PY3_DIR +} + +function init() +{ + pushd "$script_dir" || exit + PY3_DIR=$script_dir/../py3 + if [ -d $PY3_DIR ] + then + echo "Reuse $PY3_DIR env" + else + create_py_env $PY3_DIR + fi + source $PY3_DIR/bin/activate + echo "Source py3 env." + popd || exit +} + # run a bunch of ut cases # param 1 could be like examples below: # raylet, java, python_core, python_non_core, streaming @@ -43,6 +65,23 @@ function test_streaming_java() popd || exit } +function test_streaming_python() +{ + pushd "$script_dir" || exit + # Avoid macos build in python2 + if [[ $OSTYPE == "darwin" ]]; then + pushd $script_dir/python + python3 setup.py install --verbose + popd + else + pip install -e python --verbose + fi + python3 -m pytest $script_dir/python/raystreaming/tests/simple --capture=no + exit $? + + popd || exit +} + function streaming_package() @@ -77,7 +116,7 @@ function run_case() fi if [[ "$test_categories" == *java* ]]; then - echo "Running package." + echo "Running java test cases." set +e test_streaming_java @@ -88,6 +127,18 @@ function run_case() fi fi + if [[ "$test_categories" == *python* ]]; then + echo "Running python test cases." + set +e + + test_streaming_python + CODE=$? + + if [[ $CODE != 0 ]]; then + exit $CODE + fi + fi + if [[ "$test_categories" == *cpp* ]]; then echo "Running cpp tests." set +e @@ -168,4 +219,5 @@ fi #if [[ "$TEST_CATEGORIES" != *lint* ]]; then # compile #fi +init ut_all $TEST_CATEGORIES diff --git a/streaming/java/checkstyle-suppressions.xml b/streaming/java/checkstyle-suppressions.xml index 123971d4..8a4fd5b4 100644 --- a/streaming/java/checkstyle-suppressions.xml +++ b/streaming/java/checkstyle-suppressions.xml @@ -12,6 +12,7 @@ + diff --git a/streaming/java/pom.xml b/streaming/java/pom.xml index 321451ea..cae6ce9c 100644 --- a/streaming/java/pom.xml +++ b/streaming/java/pom.xml @@ -63,7 +63,7 @@ 1.8 UTF-8 - 2.0.0-SNAPSHOT + 1.11.0 0.0.1 1.10.19 2.57 diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java index ee987146..ef258a7e 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java @@ -38,16 +38,14 @@ public Record deserialize(byte[] bytes) { List list = (List) msgPackSerializer.deserialize(bytes); Byte typeId = (Byte) list.get(0); switch (typeId) { - case RECORD_TYPE_ID: - { + case RECORD_TYPE_ID: { String stream = (String) list.get(1); Object value = list.get(2); Record record = new Record(value); record.setStream(stream); return record; } - case KEY_RECORD_TYPE_ID: - { + case KEY_RECORD_TYPE_ID: { String stream = (String) list.get(1); Object key = list.get(2); Object value = list.get(3); diff --git a/streaming/python/raystreaming/__init__.pxd b/streaming/python/raystreaming/__init__.pxd index e69de29b..82800bae 100644 --- a/streaming/python/raystreaming/__init__.pxd +++ b/streaming/python/raystreaming/__init__.pxd @@ -0,0 +1,4 @@ +# cython: profile=False +# distutils: language = c++ +# cython: embedsignature = True +# cython: language_level = 3 diff --git a/streaming/python/raystreaming/runtime/gateway_client.py b/streaming/python/raystreaming/runtime/gateway_client.py index 54442c29..e1d5572c 100644 --- a/streaming/python/raystreaming/runtime/gateway_client.py +++ b/streaming/python/raystreaming/runtime/gateway_client.py @@ -12,7 +12,7 @@ class GatewayClient: _PYTHON_GATEWAY_CLASSNAME = b"io.ray.streaming.runtime.python.PythonGateway" def __init__(self): - self._python_gateway_actor = ray.java_actor_class( + self._python_gateway_actor = ray.cross_language.java_actor_class( GatewayClient._PYTHON_GATEWAY_CLASSNAME ).remote() diff --git a/streaming/python/raystreaming/tests/test_function.py b/streaming/python/raystreaming/tests/simple/test_function.py similarity index 100% rename from streaming/python/raystreaming/tests/test_function.py rename to streaming/python/raystreaming/tests/simple/test_function.py diff --git a/streaming/python/raystreaming/tests/test_operator.py b/streaming/python/raystreaming/tests/simple/test_operator.py similarity index 100% rename from streaming/python/raystreaming/tests/test_operator.py rename to streaming/python/raystreaming/tests/simple/test_operator.py diff --git a/streaming/python/raystreaming/tests/test_serialization.py b/streaming/python/raystreaming/tests/simple/test_serialization.py similarity index 74% rename from streaming/python/raystreaming/tests/test_serialization.py rename to streaming/python/raystreaming/tests/simple/test_serialization.py index 28d6acc5..d18967f6 100644 --- a/streaming/python/raystreaming/tests/test_serialization.py +++ b/streaming/python/raystreaming/tests/simple/test_serialization.py @@ -1,5 +1,5 @@ -from ray.streaming.runtime.serialization import CrossLangSerializer -from ray.streaming.message import Record, KeyRecord +from raystreaming.runtime.serialization import CrossLangSerializer +from raystreaming.message import Record, KeyRecord def test_serialize():