Added unique namespace per test

SeldonIO · Nov 14, 2019 · 1727f1d · 1727f1d
1 parent da27ed8
commit 1727f1d
Show file tree

Hide file tree

Showing 5 changed files with 255 additions and 325 deletions.
diff --git a/testing/scripts/seldon_utils.py b/testing/scripts/seldon_utils.py
@@ -7,12 +7,15 @@
 from k8s_utils import *
 
 
-def wait_for_rollout(deploymentName):
-    ret = run("kubectl rollout status -n test1 deploy/" + deploymentName, shell=True)
+def wait_for_rollout(deploymentName, namespace):
+    ret = run(
+        f"kubectl rollout status -n {namespace} deploy/" + deploymentName, shell=True
+    )
     while ret.returncode > 0:
         time.sleep(1)
         ret = run(
-            "kubectl rollout status -n test1 deploy/" + deploymentName, shell=True
+            f"kubectl rollout status -n {namespace} deploy/" + deploymentName,
+            shell=True,
         )
 
 

diff --git a/testing/scripts/test_helm_charts_clusterwide.py b/testing/scripts/test_helm_charts_clusterwide.py
@@ -3,48 +3,51 @@
 from k8s_utils import *
 
 
-def wait_for_shutdown(deploymentName):
-    ret = run("kubectl get -n test1 deploy/" + deploymentName, shell=True)
+def wait_for_shutdown(deploymentName, namespace):
+    ret = run(f"kubectl get -n {namespace} deploy/" + deploymentName, shell=True)
     while ret.returncode == 0:
         time.sleep(1)
-        ret = run("kubectl get -n test1 deploy/" + deploymentName, shell=True)
+        ret = run(f"kubectl get -n {namespace} deploy/" + deploymentName, shell=True)
 
 
 class TestClusterWide(object):
 
     # Test singe model helm script with 4 API methods
     def test_single_model(self):
-        run("helm delete mymodel --purge", shell=True)
+        namespace = "test-single-model"
+        run(f"kubectl create namespace {namespace}", shell=true, check=true)
         run(
-            "helm install ../../helm-charts/seldon-single-model --name mymodel --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1",
+            f"helm install ../../helm-charts/seldon-single-model --name mymodel-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}",
             shell=True,
             check=True,
         )
-        wait_for_rollout("mymodel-mymodel-7cd068f")
-        initial_rest_request("mymodel", "test1")
+        wait_for_rollout("mymodel-mymodel-7cd068f", namespace)
+        initial_rest_request("mymodel", namespace)
         print("Test Ambassador REST gateway")
-        r = rest_request_ambassador("mymodel", "test1", API_AMBASSADOR)
+        r = rest_request_ambassador("mymodel", namespace, API_AMBASSADOR)
         print(r.json())
         assert r.status_code == 200
         assert len(r.json()["data"]["tensor"]["values"]) == 1
         print("Test Ambassador gRPC gateway")
-        r = grpc_request_ambassador2("mymodel", "test1", API_AMBASSADOR)
+        r = grpc_request_ambassador2("mymodel", namespace, API_AMBASSADOR)
         print(r)
-        run("helm delete mymodel --purge", shell=True)
+        run(f"helm delete mymodel-{namespace} --purge", shell=True)
+        run(f"kubectl delete namespace {namespace}", shell=true, check=true)
 
     # Test AB Test model helm script with 4 API methods
     def test_abtest_model(self):
-        run("helm delete myabtest --purge", shell=True)
+        namespace = "test-abtest-model"
+        run(f"kubectl create namespace {namespace}", shell=true, check=true)
         run(
-            "helm install ../../helm-charts/seldon-abtest --name myabtest --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1",
+            f"helm install ../../helm-charts/seldon-abtest --name myabtest-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}",
             shell=True,
             check=True,
         )
-        wait_for_rollout("myabtest-myabtest-41de5b8")
-        wait_for_rollout("myabtest-myabtest-df66c5c")
-        initial_rest_request("myabtest", "test1")
+        wait_for_rollout("myabtest-myabtest-41de5b8", namespace)
+        wait_for_rollout("myabtest-myabtest-df66c5c", namespace)
+        initial_rest_request("myabtest", namespace)
         print("Test Ambassador REST gateway")
-        r = rest_request_ambassador("myabtest", "test1", API_AMBASSADOR)
+        r = rest_request_ambassador("myabtest", namespace, API_AMBASSADOR)
         print(r.json())
         assert r.status_code == 200
         assert len(r.json()["data"]["tensor"]["values"]) == 1
@@ -54,22 +57,24 @@ def test_abtest_model(self):
         )
         # r = grpc_request_ambassador2("myabtest", "test1", API_AMBASSADOR)
         # print(r)
-        run("helm delete myabtest --purge", shell=True)
+        run(f"helm delete myabtest-{namespace} --purge", shell=True)
+        run(f"kubectl delete namespace {namespace}", shell=true, check=true)
 
     # Test MAB Test model helm script with 4 API methods
     def test_mab_model(self):
-        run("helm delete mymab --purge", shell=True)
+        namespace = "test-mab-model"
+        run(f"kubectl create namespace {namespace}", shell=true, check=true)
         run(
-            "helm install ../../helm-charts/seldon-mab --name mymab --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace test1",
+            f"helm install ../../helm-charts/seldon-mab --name mymab-{namespace} --set oauth.key=oauth-key --set oauth.secret=oauth-secret --namespace {namespace}",
             shell=True,
             check=True,
         )
-        wait_for_rollout("mymab-mymab-41de5b8")
-        wait_for_rollout("mymab-mymab-b8038b2")
-        wait_for_rollout("mymab-mymab-df66c5c")
+        wait_for_rollout("mymab-mymab-41de5b8", namespace)
+        wait_for_rollout("mymab-mymab-b8038b2", namespace)
+        wait_for_rollout("mymab-mymab-df66c5c", namespace)
         initial_rest_request("mymab", "test1")
         print("Test Ambassador REST gateway")
-        r = rest_request_ambassador("mymab", "test1", API_AMBASSADOR)
+        r = rest_request_ambassador("mymab", namespace, API_AMBASSADOR)
         print(r.json())
         assert r.status_code == 200
         assert len(r.json()["data"]["tensor"]["values"]) == 1
@@ -79,4 +84,5 @@ def test_mab_model(self):
         )
         # r = grpc_request_ambassador2("mymab", "test1", API_AMBASSADOR)
         # print(r)
-        run("helm delete mymab --purge", shell=True)
+        run(f"helm delete mymab-{namespace} --purge", shell=True)
+        run(f"kubectl delete namespace {namespace}", shell=true, check=true)
diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py
@@ -4,10 +4,10 @@
 from seldon_core.seldon_client import SeldonClient
 
 
-def wait_for_status(name):
+def wait_for_status(name, namespace):
     for attempts in range(7):
         completedProcess = run(
-            "kubectl get sdep " + name + " -o json -n seldon",
+            f"kubectl get sdep " + name + " -o json -n {namespace}",
             shell=True,
             check=True,
             stdout=subprocess.PIPE,
@@ -21,59 +21,84 @@ def wait_for_status(name):
             time.sleep(5)
 
 
-def wait_for_rollout(deploymentName):
-    ret = run("kubectl rollout status deploy/" + deploymentName, shell=True)
+def wait_for_rollout(deploymentName, namespace):
+    ret = run(
+        f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True
+    )
     while ret.returncode > 0:
         time.sleep(1)
-        ret = run("kubectl rollout status deploy/" + deploymentName, shell=True)
+        ret = run(
+            f"kubectl rollout status deploy/{deploymentName} -n {namespace}", shell=True
+        )
 
 
 class TestPrepack(object):
 
     # Test prepackaged server for sklearn
     def test_sklearn(self):
-        run("kubectl delete sdep --all", shell=True)
+        namespace = "test-sklearn"
+        run(f"kubectl create namespace {namespace}", shell=True, check=True)
         run(
-            "kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml",
+            f"kubectl apply -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}",
             shell=True,
             check=True,
         )
-        wait_for_rollout("iris-default-4903e3c")
-        wait_for_status("sklearn")
+        wait_for_rollout("iris-default-4903e3c", namespace)
+        wait_for_status("sklearn", namespace)
         print("Initial request")
-        sc = SeldonClient(deployment_name="sklearn", namespace="seldon")
+        sc = SeldonClient(deployment_name="sklearn", namespace=namespace)
         r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4))
         assert r.success
         print("Success for test_prepack_sklearn")
+        run(
+            f"kubectl delete -f ../../servers/sklearnserver/samples/iris.yaml -n {namespace}",
+            shell=True,
+            check=True,
+        )
+        run(f"kubectl delete namespace {namespace}", shell=True, check=True)
 
     # Test prepackaged server for tfserving
     def test_tfserving(self):
-        run("kubectl delete sdep --all", shell=True)
+        namespace = "test-tfserving"
+        run(f"kubectl create namespace {namespace}", shell=True, check=True)
         run(
-            "kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml",
+            f"kubectl apply -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}",
             shell=True,
             check=True,
         )
-        wait_for_rollout("mnist-default-725903e")
-        wait_for_status("tfserving")
+        wait_for_rollout("mnist-default-725903e", namespace)
+        wait_for_status("tfserving", namespace)
         print("Initial request")
-        sc = SeldonClient(deployment_name="tfserving", namespace="seldon")
+        sc = SeldonClient(deployment_name="tfserving", namespace=namespace)
         r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 784))
         assert r.success
         print("Success for test_prepack_tfserving")
+        run(
+            f"kubectl delete -f ../../servers/tfserving/samples/mnist_rest.yaml -n {namespace}",
+            shell=True,
+            check=True,
+        )
+        run(f"kubectl delete namespace {namespace}", shell=True, check=True)
 
     # Test prepackaged server for xgboost
     def test_xgboost(self):
-        run("kubectl delete sdep --all", shell=True)
+        namespace = "test-xgboost"
+        run(f"kubectl create namespace {namespace}", shell=True, check=True)
         run(
-            "kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml",
+            f"kubectl apply -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}",
             shell=True,
             check=True,
         )
-        wait_for_rollout("iris-default-af1783b")
-        wait_for_status("xgboost")
+        wait_for_rollout("iris-default-af1783b", namespace)
+        wait_for_status("xgboost", namespace)
         print("Initial request")
-        sc = SeldonClient(deployment_name="xgboost", namespace="seldon")
+        sc = SeldonClient(deployment_name="xgboost", namespace=namespace)
         r = sc.predict(gateway="ambassador", transport="rest", shape=(1, 4))
         assert r.success
         print("Success for test_prepack_xgboost")
+        run(
+            f"kubectl delete -f ../../servers/xgboostserver/samples/iris.yaml -n {namespace}",
+            shell=True,
+            check=True,
+        )
+        run(f"kubectl delete namespace {namespace}", shell=True, check=True)