Merge pull request intel-analytics#4712 from yushan111/mv-orca-test

Move orca/src/test to orca/test in BigDL2.0

python/orca/test/bigdl/orca/ray/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/orca/test/bigdl/orca/ray/integration/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/orca/test/bigdl/orca/ray/integration/ray_on_yarn.py

@@ -0,0 +1,72 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import ray
+
+from bigdl.dllib.utils.nncontext import init_spark_on_yarn
+from bigdl.orca.ray import RayContext
+
+slave_num = 2
+
+sc = init_spark_on_yarn(
+    hadoop_conf="/opt/work/almaren-yarn-config/",
+    conda_name="ray_train",
+    num_executors=slave_num,
+    executor_cores=28,
+    executor_memory="10g",
+    driver_memory="2g",
+    driver_cores=4,
+    extra_executor_memory_for_ray="30g",
+    conf={"hello": "world"})
+
+ray_ctx = RayContext(sc=sc,
+                     object_store_memory="25g",
+                     extra_params={"temp-dir": "/tmp/hello/"},
+                     env={"http_proxy": "http://child-prc.intel.com:913",
+                          "http_proxys": "http://child-prc.intel.com:913"})
+ray_ctx.init()
+
+
+@ray.remote
+class TestRay():
+    def hostname(self):
+        import socket
+        return socket.gethostname()
+
+    def check_cv2(self):
+        # conda install -c conda-forge opencv==3.4.2
+        import cv2
+        return cv2.__version__
+
+    def ip(self):
+        return ray._private.services.get_node_ip_address()
+
+    def network(self):
+        from urllib.request import urlopen
+        try:
+            urlopen('http://www.baidu.com', timeout=3)
+            return True
+        except Exception as err:
+            return False
+
+
+actors = [TestRay.remote() for i in range(0, slave_num)]
+print(ray.get([actor.hostname.remote() for actor in actors]))
+print(ray.get([actor.ip.remote() for actor in actors]))
+# print(ray.get([actor.network.remote() for actor in actors]))
+
+ray_ctx.stop()

python/orca/test/bigdl/orca/ray/integration/test_yarn_reinit_raycontext.py

@@ -0,0 +1,58 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import time
+
+import numpy as np
+import ray
+
+from bigdl.dllib.utils.nncontext import init_spark_on_yarn
+from bigdl.orca.ray import RayContext
+
+np.random.seed(1337)  # for reproducibility
+
+
+@ray.remote
+class TestRay:
+    def hostname(self):
+        import socket
+        return socket.gethostname()
+
+
+node_num = 4
+sc = init_spark_on_yarn(
+    hadoop_conf="/opt/work/hadoop-2.7.2/etc/hadoop/",
+    conda_name="rayexample",
+    num_executors=node_num,
+    executor_cores=28,
+    executor_memory="10g",
+    driver_memory="2g",
+    driver_cores=4,
+    extra_executor_memory_for_ray="30g")
+ray_ctx = RayContext(sc=sc, object_store_memory="2g")
+ray_ctx.init()
+actors = [TestRay.remote() for i in range(0, node_num)]
+print(ray.get([actor.hostname.remote() for actor in actors]))
+ray_ctx.stop()
+# repeat
+ray_ctx = RayContext(sc=sc, object_store_memory="1g")
+ray_ctx.init()
+actors = [TestRay.remote() for i in range(0, node_num)]
+print(ray.get([actor.hostname.remote() for actor in actors]))
+ray_ctx.stop()
+
+sc.stop()
+time.sleep(3)

python/orca/test/bigdl/orca/ray/test_ray_on_local.py

@@ -0,0 +1,45 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from unittest import TestCase
+
+import pytest
+import ray
+
+from bigdl.dllib.utils.nncontext import init_spark_on_local
+from bigdl.orca.ray import RayContext
+
+
+class TestRayLocal(TestCase):
+
+    def test_local(self):
+        @ray.remote
+        class TestRay:
+            def hostname(self):
+                import socket
+                return socket.gethostname()
+
+        sc = init_spark_on_local(cores=8)
+        ray_ctx = RayContext(sc=sc, object_store_memory="1g", ray_node_cpu_cores=4)
+        address_info = ray_ctx.init()
+        assert "object_store_address" in address_info
+        actors = [TestRay.remote() for i in range(0, 4)]
+        print(ray.get([actor.hostname.remote() for actor in actors]))
+        ray_ctx.stop()
+        sc.stop()
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

python/orca/test/bigdl/orca/ray/test_reinit_raycontext.py

@@ -0,0 +1,62 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import time
+from unittest import TestCase
+
+import numpy as np
+import psutil
+import pytest
+import ray
+
+from bigdl.dllib.utils.nncontext import init_spark_on_local
+from bigdl.orca.ray import RayContext
+
+np.random.seed(1337)  # for reproducibility
+
+
+@ray.remote
+class TestRay():
+    def hostname(self):
+        import socket
+        return socket.gethostname()
+
+
+class TestUtil(TestCase):
+
+        def test_local(self):
+            node_num = 4
+            sc = init_spark_on_local(cores=node_num)
+            ray_ctx = RayContext(sc=sc, object_store_memory="1g")
+            ray_ctx.init()
+            actors = [TestRay.remote() for i in range(0, node_num)]
+            print(ray.get([actor.hostname.remote() for actor in actors]))
+            ray_ctx.stop()
+            time.sleep(3)
+            # repeat
+            print("-------------------first repeat begin!------------------")
+            ray_ctx = RayContext(sc=sc, object_store_memory="1g")
+            ray_ctx.init()
+            actors = [TestRay.remote() for i in range(0, node_num)]
+            print(ray.get([actor.hostname.remote() for actor in actors]))
+            ray_ctx.stop()
+            sc.stop()
+            time.sleep(3)
+            for process_info in ray_ctx.ray_processesMonitor.process_infos:
+                for pid in process_info.pids:
+                    assert not psutil.pid_exists(pid)
+
+if __name__ == "__main__":
+    pytest.main([__file__])

python/orca/test/bigdl/orca/ray/test_util.py

@@ -0,0 +1,33 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from unittest import TestCase
+
+import pytest
+
+import bigdl.orca.ray.utils as rutils
+
+
+class TestUtil(TestCase):
+
+    def test_resource_to_bytes(self):
+        assert 10 == rutils.resource_to_bytes("10b")
+        assert 10000 == rutils.resource_to_bytes("10k")
+        assert 10000000 == rutils.resource_to_bytes("10m")
+        assert 10000000000 == rutils.resource_to_bytes("10g")
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

python/orca/test/bigdl/orca/resources/bert/bert_config.json

@@ -0,0 +1,13 @@
+{
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "max_position_embeddings": 512,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "type_vocab_size": 2,
+  "vocab_size": 30522
+}

python/orca/test/bigdl/orca/resources/tfnet/graph_meta.json

@@ -0,0 +1 @@
+{"input_names": ["Placeholder:0"], "output_names": ["dense_1/Sigmoid:0"]}

python/orca/test/bigdl/orca/tfpark/__init__.py

@@ -0,0 +1,15 @@
+#
+# Copyright 2018 Analytics Zoo Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#