Integration tests fail intermittently

[axsaucedo]

Currently all integration tests pass when ran locally, but when these are ran in a kind cluster, they fail intermittently. Given there are occations where all tests pass (such as yesterday in #1026), and others where they fail it may be due to potentially timeouts being too slow (given that integration tests actually take longer - much longer - when ran in the cluster [approx 90 minutes])

[axsaucedo]

/assign @axsaucedo

[axsaucedo]

Example # 1 of intermittent failures (from jx get build logs "seldonio/seldon-core/pr-1019 #13 integration"):

=================================== FAILURES ===================================
_____________________ TestRollingHttp.test_rolling_update1 _____________________

self = <test_rolling_updates.TestRollingHttp object at 0x7fcd381b3b00>

    def test_rolling_update1(self):
        run("kubectl delete sdep --all", shell=True)
        wait_for_shutdown("mymodel-mymodel-e2eb561")
        run("kubectl apply -f ../resources/graph1.json", shell=True, check=True)
        wait_for_rollout("mymodel-mymodel-e2eb561")
        print("Initial request")
        r = initial_rest_request("mymodel", "seldon")
        assert r.status_code == 200
        assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
        run("kubectl apply -f ../resources/graph2.json", shell=True, check=True)
        i = 0
        for i in range(100):
            r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR)
>           assert r.status_code == 200
E           assert 500 == 200
E             -500
E             +200

test_rolling_updates.py:35: AssertionError
_____________________ TestRollingHttp.test_rolling_update2 _____________________

self = <test_rolling_updates.TestRollingHttp object at 0x7fcd383c4ac8>

    def test_rolling_update2(self):
        run("kubectl delete sdep --all", shell=True)
        wait_for_shutdown("mymodel-mymodel-e2eb561")
        run("kubectl apply -f ../resources/graph1.json", shell=True, check=True)
        wait_for_rollout("mymodel-mymodel-e2eb561")
        print("Initial request")
        r = initial_rest_request("mymodel", "seldon")
        assert r.status_code == 200
        assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
        run("kubectl apply -f ../resources/graph3.json", shell=True, check=True)
        i = 0
        for i in range(100):
            r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR)
>           assert r.status_code == 200
E           assert 503 == 200
E             -503
E             +200

test_rolling_updates.py:69: AssertionError
_____________________ TestPythonS2iK8s.test_combiner_rest ______________________

self = <test_s2i_python.TestPythonS2iK8s object at 0x7fcd382a5630>
s2i_python_version = '0.13-SNAPSHOT'

    def test_combiner_rest(self, s2i_python_version):
        tester = S2IK8S()
>       tester.test_combiner_rest(s2i_python_version)

test_s2i_python.py:126:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
test_s2i_python.py:214: in test_combiner_rest
    check=True,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

input = None, timeout = None, check = True
popenargs = ('kubectl apply -f ../resources/s2i_python_combiner.json',)
kwargs = {'shell': True}, process = <subprocess.Popen object at 0x7fcd382a5b38>
stdout = None, stderr = None, retcode = 1

    def run(*popenargs, input=None, timeout=None, check=False, **kwargs):
        """Run command with arguments and return a CompletedProcess instance.

        The returned instance will have attributes args, returncode, stdout and
        stderr. By default, stdout and stderr are not captured, and those attributes
        will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them.

        If check is True and the exit code was non-zero, it raises a
        CalledProcessError. The CalledProcessError object will have the return code
        in the returncode attribute, and output & stderr attributes if those streams
        were captured.

        If timeout is given, and the process takes too long, a TimeoutExpired
        exception will be raised.

        There is an optional argument "input", allowing you to
        pass a string to the subprocess's stdin.  If you use this argument
        you may not also use the Popen constructor's "stdin" argument, as
        it will be used internally.

        The other arguments are the same as for the Popen constructor.

        If universal_newlines=True is passed, the "input" argument must be a
        string and stdout/stderr in the returned object will be strings rather than
        bytes.
        """
        if input is not None:
            if 'stdin' in kwargs:
                raise ValueError('stdin and input arguments may not both be used.')
            kwargs['stdin'] = PIPE

        with Popen(*popenargs, **kwargs) as process:
            try:
                stdout, stderr = process.communicate(input, timeout=timeout)
            except TimeoutExpired:
                process.kill()
                stdout, stderr = process.communicate()
                raise TimeoutExpired(process.args, timeout, output=stdout,
                                     stderr=stderr)
            except:
                process.kill()
                process.wait()
                raise
            retcode = process.poll()
            if check and retcode:
                raise CalledProcessError(retcode, process.args,
>                                        output=stdout, stderr=stderr)
E               subprocess.CalledProcessError: Command 'kubectl apply -f ../resources/s2i_python_combiner.json' returned non-zero exit status 1.

/usr/local/lib/python3.6/subprocess.py:418: CalledProcessError
================== 3 failed, 28 passed in 3601.70s (1:00:01) ===================

[axsaucedo]

Example number # 2 from jx get build logs "seldonio/seldon-core/pr-1019 #12 integration":

=================================== FAILURES ===================================
_____________________ TestRollingHttp.test_rolling_update4 _____________________

self = <test_rolling_updates.TestRollingHttp object at 0x7fe088b95be0>

    def test_rolling_update4(self):
        run("kubectl delete sdep --all", shell=True)
        wait_for_shutdown("mymodel-mymodel-e2eb561")
        run("kubectl apply -f ../resources/graph1.json", shell=True, check=True)
        wait_for_rollout("mymodel-mymodel-e2eb561")
        print("Initial request")
        r = initial_rest_request("mymodel", "seldon")
        assert r.status_code == 200
        assert r.json()["data"]["tensor"]["values"] == [1.0, 2.0, 3.0, 4.0]
        run("kubectl apply -f ../resources/graph5.json", shell=True, check=True)
        i = 0
        for i in range(50):
            r = rest_request_ambassador("mymodel", "seldon", API_AMBASSADOR)
>           assert r.status_code == 200
E           assert 500 == 200
E             -500
E             +200

test_rolling_updates.py:133: AssertionError
================== 1 failed, 30 passed in 2489.32s (0:41:29) ===================

[axsaucedo]

It seems that specifically in the test_combiner_rest test, in jx get build logs "seldonio/seldon-core/pr-1019 #13 integration", the error shown is: Error from server (InternalError): error when creating "../resources/s2i_python_combiner.json": Internal error occurred: failed calling webhook "mseldondeployment.kb.io": Post https://seldon-webhook-service.seldon-system.svc:443/mutate-machinelearning-seldon-io-v1alpha2-seldondeployment?timeout=30s: dial tcp 10.97.147.71:443: connect: connection refused

Full log:

test_s2i_python.py::TestPythonS2iK8s::test_combiner_rest No resources found
---> Installing application source...
Build completed successfully
time="13:44:16" level=debug msg="Running: /usr/bin/docker [docker image inspect -f {{ .Id }} seldonio/testmodel_rest:0.1]"
time="13:44:16" level=debug msg="Running: /usr/bin/docker [docker ps -q -a --no-trunc --filter label=io.k8s.sigs.kind.cluster --format {{.Names}}\\t{{.Label \"io.k8s.sigs.kind.cluster\"}}]"
time="13:44:17" level=debug msg="Running: /usr/bin/docker [docker ps -q -a --no-trunc --filter label=io.k8s.sigs.kind.cluster --format {{.Names}}\\t{{.Label \"io.k8s.sigs.kind.cluster\"}} --filter label=io.k8s.sigs.kind.cluster=kind]"
time="13:44:17" level=debug msg="Running: /usr/bin/docker [docker inspect -f {{index .Config.Labels \"io.k8s.sigs.kind.role\"}} kind-worker]"
time="13:44:17" level=debug msg="Running: /usr/bin/docker [docker inspect -f {{index .Config.Labels \"io.k8s.sigs.kind.role\"}} kind-control-plane]"
time="13:44:17" level=debug msg="Running: /usr/bin/docker [docker exec --privileged kind-worker crictl inspecti seldonio/testmodel_rest:0.1]"
time="13:44:18" level=debug msg="Image: \"seldonio/testmodel_rest:0.1\" with ID \"sha256:901fb35985ed13411db4ac46b7ff8cac4e9fc2fa6f90867ba4697339a868086e\" not present on node \"kind-worker\""
time="13:44:18" level=debug msg="Running: /usr/bin/docker [docker exec --privileged kind-control-plane crictl inspecti seldonio/testmodel_rest:0.1]"
time="13:44:18" level=debug msg="Image: \"seldonio/testmodel_rest:0.1\" with ID \"sha256:901fb35985ed13411db4ac46b7ff8cac4e9fc2fa6f90867ba4697339a868086e\" not present on node \"kind-control-plane\""
time="13:44:18" level=debug msg="Running: /usr/bin/docker [docker save -o /tmp/image-tar958203222/image.tar seldonio/testmodel_rest:0.1]"
time="13:47:16" level=debug msg="Running: /usr/bin/docker [docker exec --privileged -i kind-worker ctr --namespace=k8s.io images import -]"
time="13:47:16" level=debug msg="Running: /usr/bin/docker [docker exec --privileged -i kind-control-plane ctr --namespace=k8s.io images import -]"
---> Installing application source...
Build completed successfully
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker image inspect -f {{ .Id }} seldonio/testcombiner_rest:0.1]"
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker ps -q -a --no-trunc --filter label=io.k8s.sigs.kind.cluster --format {{.Names}}\\t{{.Label \"io.k8s.sigs.kind.cluster\"}}]"
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker ps -q -a --no-trunc --filter label=io.k8s.sigs.kind.cluster --format {{.Names}}\\t{{.Label \"io.k8s.sigs.kind.cluster\"}} --filter label=io.k8s.sigs.kind.cluster=kind]"
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker inspect -f {{index .Config.Labels \"io.k8s.sigs.kind.role\"}} kind-worker]"
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker inspect -f {{index .Config.Labels \"io.k8s.sigs.kind.role\"}} kind-control-plane]"
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker exec --privileged kind-worker crictl inspecti seldonio/testcombiner_rest:0.1]"
time="13:52:09" level=debug msg="Image: \"seldonio/testcombiner_rest:0.1\" with ID \"sha256:8855b8bfdf219d7abc4f6e5711111f742f8ea84b964ffb34635e382c0c2a8024\" not present on node \"kind-worker\""
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker exec --privileged kind-control-plane crictl inspecti seldonio/testcombiner_rest:0.1]"
time="13:52:09" level=debug msg="Image: \"seldonio/testcombiner_rest:0.1\" with ID \"sha256:8855b8bfdf219d7abc4f6e5711111f742f8ea84b964ffb34635e382c0c2a8024\" not present on node \"kind-control-plane\""
time="13:52:09" level=debug msg="Running: /usr/bin/docker [docker save -o /tmp/image-tar257627535/image.tar seldonio/testcombiner_rest:0.1]"
time="13:55:28" level=debug msg="Running: /usr/bin/docker [docker exec --privileged -i kind-control-plane ctr --namespace=k8s.io images import -]"
time="13:55:28" level=debug msg="Running: /usr/bin/docker [docker exec --privileged -i kind-worker ctr --namespace=k8s.io images import -]"
Error from server (InternalError): error when creating "../resources/s2i_python_combiner.json": Internal error occurred: failed calling webhook "mseldondeployment.kb.io": Post https://seldon-webhook-service.seldon-system.svc:443/mutate-machinelearning-seldon-io-v1alpha2-seldondeployment?timeout=30s: dial tcp 10.97.147.71:443: connect: connection refused
cd ../s2i/python/model && s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:0.13-SNAPSHOT seldonio/testmodel_rest:0.1
kind load docker-image seldonio/testmodel_rest:0.1 --loglevel trace
cd ../s2i/python/combiner && s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:0.13-SNAPSHOT seldonio/testcombiner_rest:0.1
kind load docker-image seldonio/testcombiner_rest:0.1 --loglevel trace
FAILED

[axsaucedo]

and for the failure from `jx get build logs "seldonio/seldon-core/pr-1019 #12 integration", the test_rolling_update4 test seems to see a 503 before it returns the outputs. it's strange as you can clearly see that it returns it successfully, so it may be discarding the 200 that it sees after the 503. These may be two different errors

test_rolling_updates.py::TestRollingHttp::test_rolling_update4 seldondeployment.machinelearning.seldon.io "mymodel" deleted
NAME                      READY   UP-TO-DATE   AVAILABLE   AGE
mymodel-mymodel-e2eb561   1/1     1            1           83s
Error from server (NotFound): deployments.extensions "mymodel-mymodel-e2eb561" not found
seldondeployment.machinelearning.seldon.io/mymodel created
Waiting for deployment "mymodel-mymodel-e2eb561" rollout to finish: 0 of 1 updated replicas are available...
deployment "mymodel-mymodel-e2eb561" successfully rolled out
seldondeployment.machinelearning.seldon.io/mymodel configured
Initial request
Bad status: 404
Sleeping 1 sec and trying again
Bad status: 404
Sleeping 5 sec and trying again
Bad status: 503
Sleeping 10 sec and trying again
{'meta': {'puid': 'ivibs34q09n7dh5ismr3kqtg4r', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'ifk1ce7ecvpo5pjdfh828cukrr', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'qibiaq7oslspilhl5k5pl8kbon', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'l3kf2gbgkoepd7a29pilt8chc6', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'kcnh1ttbf9c1nken5h9bbbbtlo', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'q8v9pgd1cboc63h4drave2socf', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 't4hkgdpuqpno05b1ee72otmkng', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '7hk56qqjb5s4laejj1oqb03ofu', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'jjqukro8ei3pf36o2dtovfl9mm', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'e300a2gqa2nrdc1ubqb73r8f62', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'c72v0ojfiq1a3es954cvs36j7q', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'mocf0v5pdarame6d93qupad035', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'timvfg2gak3mn3igk4tgdas576', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '661esmapa60fvh5r3eugm515tq', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'qk3frem2ismdrp3env26m7pg1o', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'akpqjecke6aa6qfje0ctfb79u9', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'db4fr7riij494780u6ot2efspk', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '4nohl4mc0nm2d1ako1p1bnqhjo', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'ha2r62rhksj1io9lsegb59kdv1', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'i9nbi4p3kjvd13mcv7t93h41fi', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'f4ju1u2ccqo7572oc3dim1e6fl', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'pk4r8act575hhjnvtq9j1tsl9s', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'qrj8tqjjctb34phj646fmq9no9', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'n83kslu9b24hrjm20d9k0d7dal', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '7m6r0pocqofi3a5tso3niraf9m', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'otkbpkfdb0g9gjqignvcofavso', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'kb712e0qaanqplvmk8uv8hnsju', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'rge58rs5ag0of5ugbggd8lj3l5', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'm8756n7ad9rmf4k54g6pdgm7j3', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'tuu4mot084iij8gume8gdofcff', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'df690i0600scv66koid2jpljo6', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '7eic0uah2kb46ige27o5hebgd8', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'lvnei7plfjamqkfofflf6k8c4l', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '5nth3vpaffutgaajvm334131cs', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'l110dkpgenl8ejtj7iv5u9oou9', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '477dfkifjbh64g1f6kdci62j5v', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'angpv64d07tni6pg9b7pdfpr5e', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': '8413jv7vg0o68ib0lta2mjkd6f', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'hj77krrnfk9ppv84dvjcjoahgn', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'm6lrpke86kh55jvtan34clltbg', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'lfbjavjpefufosrkhrasiluo08', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'de8puvua5b1995ltdn2hprmqda', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'f1pvksqs2hm6giejngrvjmtfh2', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'e85nt7hsem6952gk6m5np6jh89', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
{'meta': {'puid': 'mj99jjv94a3kof0v4v66qu79p5', 'tags': {}, 'routing': {}, 'requestPath': {'complex-model': 'seldonio/fixed-model:0.1'}, 'metrics': []}, 'data': {'names': [], 'tensor': {'shape': [4], 'values': [1.0, 2.0, 3.0, 4.0]}}}
FAILED