[Python] Python build fix

.github/workflows/macos_building.yml

@@ -7,7 +7,7 @@ on:
     branches: [ master ]
 
 jobs:
-  streaming-cpp-test:
+  streaming-mac-pipeline:
     timeout-minutes: 60
     runs-on: ${{ matrix.os }}
     strategy:
@@ -29,5 +29,18 @@ jobs:
             fi
             sudo sh scripts/install-bazel.sh
 
-    - name: Build streaming and test
+            #- name: Install ray wheel
+            #  run: |
+            #       which python3
+            #       python3 --version
+            #       which python3.8
+            #       python3.8 -m pip3 install https://ray-mobius-us.oss-us-west-1.aliyuncs.com/ci/macos/ray-2.0.0.dev0-cp38-cp38-macosx_11_0_x86_64.whl
+
+            #- name: Streaming python test
+            #  run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_python"
+
+    - name: Streaming java test
+      run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_java"
+
+    - name: Streaming cpp test
       run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_cpp"

.github/workflows/ubuntu_building.yml

@@ -7,7 +7,7 @@ on:
     branches: [ master ]
 
 jobs:
-  streaming-cpp-test:
+  streaming-ubuntu-pipeline:
     timeout-minutes: 60
     runs-on: ubuntu-latest
     container: docker.io/library/ubuntu:latest
@@ -18,9 +18,20 @@ jobs:
     - name: Install bazel
       run:  |
             apt-get update
-            apt-get install -yq wget gcc g++ openjdk-8-jdk python3.7 zlib1g-dev zip
+            apt-get install -yq wget gcc g++ openjdk-8-jdk python3.8 zlib1g-dev zip
+            apt-get install -yq pip
             sh scripts/install-bazel.sh
 
+    - name: Install ray wheel
+      run: |
+           python3 -m pip install virtualenv
+           python3 -m virtualenv -p python3 py3
+           . py3/bin/activate
+           pip install pytest
+           pip install https://ray-mobius-us.oss-us-west-1.aliyuncs.com/ci/linux/ubuntu/ray-2.0.0.dev0-cp38-cp38-linux_x86_64.whl
+    - name: Streaming python test
+      run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_python"
+
     - name: Streaming java test
       run: sh -c "bash streaming/buildtest.sh --test_categories=streaming_java"
 

streaming/buildtest.sh

@@ -1,6 +1,28 @@
 #!/bin/bash
 script_dir=$(cd "$(dirname "${BASH_SOURCE:-$0}")" || exit; pwd)
 
+function create_py_env()
+{
+    PY3_DIR=${1}
+    python3 -m pip install virtualenv
+    python3 -m virtualenv -p python3 $PY3_DIR
+}
+
+function init()
+{
+    pushd "$script_dir" || exit
+    PY3_DIR=$script_dir/../py3
+    if [ -d $PY3_DIR ]
+    then
+        echo "Reuse $PY3_DIR env"
+    else
+        create_py_env $PY3_DIR
+    fi
+    source $PY3_DIR/bin/activate
+    echo "Source py3 env."
+    popd || exit
+}
+
 # run a bunch of ut cases
 # param 1 could be like examples below：
 # raylet, java, python_core, python_non_core, streaming
@@ -43,6 +65,23 @@ function test_streaming_java()
     popd || exit
 }
 
+function test_streaming_python() 
+{
+    pushd "$script_dir" || exit
+    # Avoid macos build in python2
+    if [[ $OSTYPE == "darwin" ]]; then
+        pushd $script_dir/python
+        python3 setup.py install --verbose
+        popd
+    else
+        pip install -e python --verbose 
+    fi
+    python3 -m pytest $script_dir/python/raystreaming/tests/simple --capture=no
+    exit $?
+
+    popd || exit
+}
+
 
 
 function streaming_package() 
@@ -77,7 +116,7 @@ function run_case()
     fi
 
     if [[ "$test_categories" == *java* ]]; then
-      echo "Running package."
+      echo "Running java test cases."
       set +e
 
       test_streaming_java
@@ -88,6 +127,18 @@ function run_case()
       fi
     fi
 
+    if [[ "$test_categories" == *python* ]]; then
+      echo "Running python test cases."
+      set +e
+
+      test_streaming_python
+      CODE=$?
+
+      if [[ $CODE != 0 ]]; then
+        exit $CODE
+      fi
+    fi
+
     if [[ "$test_categories" == *cpp* ]]; then
       echo "Running cpp tests."
       set +e
@@ -168,4 +219,5 @@ fi
 #if [[ "$TEST_CATEGORIES" != *lint* ]]; then
 #  compile
 #fi
+init
 ut_all $TEST_CATEGORIES

streaming/java/checkstyle-suppressions.xml

@@ -12,6 +12,7 @@
   <suppress files=".*" checks="MissingJavadocMethod"/>
   <suppress files=".*" checks="MissingJavadocType"/>
   <suppress files=".*" checks="InvalidJavadocPosition"/>
+  <suppress files=".*" checks="Indentation"/>
   <suppress files=".*" checks="RequireEmptyLineBeforeBlockTagGroup"/>
   <suppress files=".*" checks="SingleLineJavadoc"/>
   <suppress files=".*" checks="VariableDeclarationUsageDistance"/>

streaming/java/pom.xml

@@ -63,7 +63,7 @@
   <properties>
     <java.version>1.8</java.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <ray.version>2.0.0-SNAPSHOT</ray.version>
+    <ray.version>1.11.0</ray.version>
     <streaming.version>0.0.1</streaming.version>
     <mockito.version>1.10.19</mockito.version>
     <fst.version>2.57</fst.version>

streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java

@@ -38,16 +38,14 @@ public Record deserialize(byte[] bytes) {
     List list = (List) msgPackSerializer.deserialize(bytes);
     Byte typeId = (Byte) list.get(0);
     switch (typeId) {
-      case RECORD_TYPE_ID:
-        {
+      case RECORD_TYPE_ID: {
           String stream = (String) list.get(1);
           Object value = list.get(2);
           Record record = new Record(value);
           record.setStream(stream);
           return record;
         }
-      case KEY_RECORD_TYPE_ID:
-        {
+      case KEY_RECORD_TYPE_ID: {
           String stream = (String) list.get(1);
           Object key = list.get(2);
           Object value = list.get(3);

streaming/python/raystreaming/__init__.pxd

@@ -0,0 +1,4 @@
+# cython: profile=False
+# distutils: language = c++
+# cython: embedsignature = True
+# cython: language_level = 3

streaming/python/raystreaming/runtime/gateway_client.py

@@ -12,7 +12,7 @@ class GatewayClient:
     _PYTHON_GATEWAY_CLASSNAME = b"io.ray.streaming.runtime.python.PythonGateway"
 
     def __init__(self):
-        self._python_gateway_actor = ray.java_actor_class(
+        self._python_gateway_actor = ray.cross_language.java_actor_class(
             GatewayClient._PYTHON_GATEWAY_CLASSNAME
         ).remote()
 

streaming/python/raystreaming/tests/test_serialization.py → streaming/python/raystreaming/tests/simple/test_serialization.py

@@ -1,5 +1,5 @@
-from ray.streaming.runtime.serialization import CrossLangSerializer
-from ray.streaming.message import Record, KeyRecord
+from raystreaming.runtime.serialization import CrossLangSerializer
+from raystreaming.message import Record, KeyRecord
 
 
 def test_serialize():