diff --git a/.github/workflows/macos_building.yml b/.github/workflows/macos_building.yml
index 561c94cc..95e0c36c 100644
--- a/.github/workflows/macos_building.yml
+++ b/.github/workflows/macos_building.yml
@@ -7,7 +7,7 @@ on:
branches: [ master ]
jobs:
- streaming-cpp-test:
+ streaming-mac-pipeline:
timeout-minutes: 60
runs-on: ${{ matrix.os }}
strategy:
@@ -29,5 +29,18 @@ jobs:
fi
sudo sh scripts/install-bazel.sh
- - name: Build streaming and test
+ #- name: Install ray wheel
+ # run: |
+ # which python3
+ # python3 --version
+ # which python3.8
+ # python3.8 -m pip3 install https://ray-mobius-us.oss-us-west-1.aliyuncs.com/ci/macos/ray-2.0.0.dev0-cp38-cp38-macosx_11_0_x86_64.whl
+
+ #- name: Streaming python test
+ # run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_python"
+
+ - name: Streaming java test
+ run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_java"
+
+ - name: Streaming cpp test
run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_cpp"
diff --git a/.github/workflows/ubuntu_building.yml b/.github/workflows/ubuntu_building.yml
index 2da68386..38b835bb 100644
--- a/.github/workflows/ubuntu_building.yml
+++ b/.github/workflows/ubuntu_building.yml
@@ -7,7 +7,7 @@ on:
branches: [ master ]
jobs:
- streaming-cpp-test:
+ streaming-ubuntu-pipeline:
timeout-minutes: 60
runs-on: ubuntu-latest
container: docker.io/library/ubuntu:latest
@@ -18,9 +18,20 @@ jobs:
- name: Install bazel
run: |
apt-get update
- apt-get install -yq wget gcc g++ openjdk-8-jdk python3.7 zlib1g-dev zip
+ apt-get install -yq wget gcc g++ openjdk-8-jdk python3.8 zlib1g-dev zip
+ apt-get install -yq pip
sh scripts/install-bazel.sh
+ - name: Install ray wheel
+ run: |
+ python3 -m pip install virtualenv
+ python3 -m virtualenv -p python3 py3
+ . py3/bin/activate
+ pip install pytest
+ pip install https://ray-mobius-us.oss-us-west-1.aliyuncs.com/ci/linux/ubuntu/ray-2.0.0.dev0-cp38-cp38-linux_x86_64.whl
+ - name: Streaming python test
+ run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_python"
+
- name: Streaming java test
run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_java"
diff --git a/streaming/buildtest.sh b/streaming/buildtest.sh
index 07f72f1b..24646057 100755
--- a/streaming/buildtest.sh
+++ b/streaming/buildtest.sh
@@ -1,6 +1,28 @@
#!/bin/bash
script_dir=$(cd "$(dirname "${BASH_SOURCE:-$0}")" || exit; pwd)
+function create_py_env()
+{
+ PY3_DIR=${1}
+ python3 -m pip install virtualenv
+ python3 -m virtualenv -p python3 $PY3_DIR
+}
+
+function init()
+{
+ pushd "$script_dir" || exit
+ PY3_DIR=$script_dir/../py3
+ if [ -d $PY3_DIR ]
+ then
+ echo "Reuse $PY3_DIR env"
+ else
+ create_py_env $PY3_DIR
+ fi
+ source $PY3_DIR/bin/activate
+ echo "Source py3 env."
+ popd || exit
+}
+
# run a bunch of ut cases
# param 1 could be like examples below:
# raylet, java, python_core, python_non_core, streaming
@@ -43,6 +65,23 @@ function test_streaming_java()
popd || exit
}
+function test_streaming_python()
+{
+ pushd "$script_dir" || exit
+ # Avoid macos build in python2
+ if [[ $OSTYPE == "darwin" ]]; then
+ pushd $script_dir/python
+ python3 setup.py install --verbose
+ popd
+ else
+ pip install -e python --verbose
+ fi
+ python3 -m pytest $script_dir/python/raystreaming/tests/simple --capture=no
+ exit $?
+
+ popd || exit
+}
+
function streaming_package()
@@ -77,7 +116,7 @@ function run_case()
fi
if [[ "$test_categories" == *java* ]]; then
- echo "Running package."
+ echo "Running java test cases."
set +e
test_streaming_java
@@ -88,6 +127,18 @@ function run_case()
fi
fi
+ if [[ "$test_categories" == *python* ]]; then
+ echo "Running python test cases."
+ set +e
+
+ test_streaming_python
+ CODE=$?
+
+ if [[ $CODE != 0 ]]; then
+ exit $CODE
+ fi
+ fi
+
if [[ "$test_categories" == *cpp* ]]; then
echo "Running cpp tests."
set +e
@@ -168,4 +219,5 @@ fi
#if [[ "$TEST_CATEGORIES" != *lint* ]]; then
# compile
#fi
+init
ut_all $TEST_CATEGORIES
diff --git a/streaming/java/checkstyle-suppressions.xml b/streaming/java/checkstyle-suppressions.xml
index 123971d4..8a4fd5b4 100644
--- a/streaming/java/checkstyle-suppressions.xml
+++ b/streaming/java/checkstyle-suppressions.xml
@@ -12,6 +12,7 @@
+
diff --git a/streaming/java/pom.xml b/streaming/java/pom.xml
index 321451ea..cae6ce9c 100644
--- a/streaming/java/pom.xml
+++ b/streaming/java/pom.xml
@@ -63,7 +63,7 @@
1.8
UTF-8
- 2.0.0-SNAPSHOT
+ 1.11.0
0.0.1
1.10.19
2.57
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java
index ee987146..ef258a7e 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java
@@ -38,16 +38,14 @@ public Record deserialize(byte[] bytes) {
List list = (List) msgPackSerializer.deserialize(bytes);
Byte typeId = (Byte) list.get(0);
switch (typeId) {
- case RECORD_TYPE_ID:
- {
+ case RECORD_TYPE_ID: {
String stream = (String) list.get(1);
Object value = list.get(2);
Record record = new Record(value);
record.setStream(stream);
return record;
}
- case KEY_RECORD_TYPE_ID:
- {
+ case KEY_RECORD_TYPE_ID: {
String stream = (String) list.get(1);
Object key = list.get(2);
Object value = list.get(3);
diff --git a/streaming/python/raystreaming/__init__.pxd b/streaming/python/raystreaming/__init__.pxd
index e69de29b..82800bae 100644
--- a/streaming/python/raystreaming/__init__.pxd
+++ b/streaming/python/raystreaming/__init__.pxd
@@ -0,0 +1,4 @@
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+# cython: language_level = 3
diff --git a/streaming/python/raystreaming/runtime/gateway_client.py b/streaming/python/raystreaming/runtime/gateway_client.py
index 54442c29..e1d5572c 100644
--- a/streaming/python/raystreaming/runtime/gateway_client.py
+++ b/streaming/python/raystreaming/runtime/gateway_client.py
@@ -12,7 +12,7 @@ class GatewayClient:
_PYTHON_GATEWAY_CLASSNAME = b"io.ray.streaming.runtime.python.PythonGateway"
def __init__(self):
- self._python_gateway_actor = ray.java_actor_class(
+ self._python_gateway_actor = ray.cross_language.java_actor_class(
GatewayClient._PYTHON_GATEWAY_CLASSNAME
).remote()
diff --git a/streaming/python/raystreaming/tests/test_function.py b/streaming/python/raystreaming/tests/simple/test_function.py
similarity index 100%
rename from streaming/python/raystreaming/tests/test_function.py
rename to streaming/python/raystreaming/tests/simple/test_function.py
diff --git a/streaming/python/raystreaming/tests/test_operator.py b/streaming/python/raystreaming/tests/simple/test_operator.py
similarity index 100%
rename from streaming/python/raystreaming/tests/test_operator.py
rename to streaming/python/raystreaming/tests/simple/test_operator.py
diff --git a/streaming/python/raystreaming/tests/test_serialization.py b/streaming/python/raystreaming/tests/simple/test_serialization.py
similarity index 74%
rename from streaming/python/raystreaming/tests/test_serialization.py
rename to streaming/python/raystreaming/tests/simple/test_serialization.py
index 28d6acc5..d18967f6 100644
--- a/streaming/python/raystreaming/tests/test_serialization.py
+++ b/streaming/python/raystreaming/tests/simple/test_serialization.py
@@ -1,5 +1,5 @@
-from ray.streaming.runtime.serialization import CrossLangSerializer
-from ray.streaming.message import Record, KeyRecord
+from raystreaming.runtime.serialization import CrossLangSerializer
+from raystreaming.message import Record, KeyRecord
def test_serialize():