From 8a47df5ae403130faba2e2eefc69b39594d6e72e Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Wed, 3 Jun 2020 19:59:45 +0100 Subject: [PATCH] Added update to use pvc instead of param --- examples/batch/argo-workflows/README.ipynb | 5705 ++++++++++++++++- .../README_saved_from_latest.ipynb | 459 ++ .../seldon-batch-workflow/Chart.yaml | 9 + .../seldon-batch-workflow/README.md | 3 + .../templates/workflow.yaml | 88 + .../seldon-batch-workflow/values.yaml | 45 + .../templates/workflow.yaml | 51 +- .../seldon-batch-workflow/values.yaml | 7 + 8 files changed, 6305 insertions(+), 62 deletions(-) create mode 100644 examples/batch/argo-workflows/README_saved_from_latest.ipynb create mode 100644 examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/Chart.yaml create mode 100644 examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/README.md create mode 100644 examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/templates/workflow.yaml create mode 100644 examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/values.yaml diff --git a/examples/batch/argo-workflows/README.ipynb b/examples/batch/argo-workflows/README.ipynb index 548d04b5f5..d77941786e 100644 --- a/examples/batch/argo-workflows/README.ipynb +++ b/examples/batch/argo-workflows/README.ipynb @@ -685,7 +685,7 @@ }, { "cell_type": "code", - "execution_count": 484, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -703,7 +703,7 @@ }, { "cell_type": "code", - "execution_count": 485, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -738,15 +738,15 @@ }, { "cell_type": "code", - "execution_count": 486, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[m\u001b[32;1mBucket created successfully `minio-local/data`.\u001b[0m\n", - "...-data.txt: 614.15 KiB / 614.15 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 9.74 MiB/s 0s\u001b[0m\u001b[0m\u001b[m\u001b[32;1m" + "\u001b[33;3mmc: \u001b[0m\u001b[33;3mUnable to make bucket `minio-local/data`. Your previous request to create the named bucket succeeded and you already own it.\n", + "...-data.txt: 614.15 KiB / 614.15 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 5.67 MiB/s 0s\u001b[0m\u001b[0m\u001b[m\u001b[32;1m" ] } ], @@ -770,7 +770,7 @@ }, { "cell_type": "code", - "execution_count": 683, + "execution_count": 48, "metadata": { "scrolled": false }, @@ -783,7 +783,7 @@ "Namespace: default\r\n", "ServiceAccount: default\r\n", "Status: Pending\r\n", - "Created: Sat May 02 12:06:45 +0100 (now)\r\n" + "Created: Wed Jun 03 18:46:13 +0100 (now)\r\n" ] } ], @@ -791,14 +791,14 @@ "!helm template seldon-batch-workflow helm-charts/seldon-batch-workflow/ \\\n", " --set workflow.name=seldon-batch-process \\\n", " --set seldonDeployment.name=sklearn \\\n", - " --set replicas 10 \\\n", - " --set workers 100 \\\n", + " --set seldonDeployment.replicas=10 \\\n", + " --set batchWorker.workers=100 \\\n", " | argo submit -" ] }, { "cell_type": "code", - "execution_count": 684, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -806,7 +806,7 @@ "output_type": "stream", "text": [ "NAME STATUS AGE DURATION PRIORITY\r\n", - "seldon-batch-process Running 2s 2s 0\r\n" + "seldon-batch-process Running 1m 1m 0\r\n" ] } ], @@ -816,7 +816,7 @@ }, { "cell_type": "code", - "execution_count": 773, + "execution_count": 55, "metadata": {}, "outputs": [ { @@ -826,19 +826,17 @@ "Name: seldon-batch-process\r\n", "Namespace: default\r\n", "ServiceAccount: default\r\n", - "Status: Succeeded\r\n", - "Created: Sat May 02 12:06:45 +0100 (1 minute ago)\r\n", - "Started: Sat May 02 12:06:45 +0100 (1 minute ago)\r\n", - "Finished: Sat May 02 12:07:54 +0100 (2 seconds ago)\r\n", - "Duration: 1 minute 9 seconds\r\n", + "Status: Running\r\n", + "Created: Wed Jun 03 18:46:13 +0100 (27 minutes ago)\r\n", + "Started: Wed Jun 03 18:46:13 +0100 (27 minutes ago)\r\n", + "Duration: 27 minutes 59 seconds\r\n", "\r\n", "\u001b[39mSTEP\u001b[0m PODNAME DURATION MESSAGE\r\n", - " \u001b[32m✔\u001b[0m seldon-batch-process (seldon-batch-process) \r\n", - " ├---\u001b[32m✔\u001b[0m create-seldon-resource (create-seldon-resource-template) seldon-batch-process-3626514072 2s \r\n", - " ├---\u001b[32m✔\u001b[0m wait-seldon-resource (wait-seldon-resource-template) seldon-batch-process-2052519094 23s \r\n", - " ├---\u001b[32m✔\u001b[0m download-object-store (download-object-store-template) seldon-batch-process-1257652469 2s \r\n", - " ├---\u001b[32m✔\u001b[0m process-batch-inputs (process-batch-inputs-template) seldon-batch-process-2033515954 27s \r\n", - " └---\u001b[32m✔\u001b[0m upload-object-store (upload-object-store-template) seldon-batch-process-2123074048 2s \r\n" + " \u001b[36m●\u001b[0m seldon-batch-process (seldon-batch-process) \r\n", + " ├---\u001b[32m✔\u001b[0m create-seldon-resource (create-seldon-resource-template) seldon-batch-process-3626514072 3s \r\n", + " ├---\u001b[32m✔\u001b[0m wait-seldon-resource (wait-seldon-resource-template) seldon-batch-process-2052519094 30s \r\n", + " ├---\u001b[32m✔\u001b[0m download-object-store (download-object-store-template) seldon-batch-process-1257652469 4s \r\n", + " └---\u001b[36m●\u001b[0m process-batch-inputs (process-batch-inputs-template) seldon-batch-process-2033515954 27m \r\n" ] } ], @@ -848,20 +846,27 @@ }, { "cell_type": "code", - "execution_count": 774, + "execution_count": 56, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-02T11:06:46Z\" level=info msg=\"Starting Workflow Executor\" version=vv2.7.4+50b209c.dirty\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-02T11:06:46Z\" level=info msg=\"Creating a docker executor\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-02T11:06:46Z\" level=info msg=\"Executor (version: vv2.7.4+50b209c.dirty, build_date: 2020-04-16T16:37:57Z) initialized (pod: default/seldon-batch-process-3626514072) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"arguments\\\":{},\\\"inputs\\\":{},\\\"outputs\\\":{},\\\"metadata\\\":{},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n namespace: default\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-batch-process\\\\\\\"\\\\n uid: \\\\\\\"72ad4430-c70a-4976-8043-2ba6c2f56772\\\\\\\"\\\\nspec:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n predictors:\\\\n - graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 10\\\\n \\\\n\\\"}}\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-02T11:06:46Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-02T11:06:46Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-02T11:06:47Z\" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-02T11:06:47Z\" level=info msg=\"No output parameters\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-06-03T17:46:15Z\" level=info msg=\"Starting Workflow Executor\" version=v2.8.0-rc4+8f69617.dirty\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-06-03T17:46:15Z\" level=info msg=\"Creating a docker executor\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-06-03T17:46:15Z\" level=info msg=\"Executor (version: v2.8.0-rc4+8f69617.dirty, build_date: 2020-05-12T15:17:15Z) initialized (pod: default/seldon-batch-process-3626514072) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"arguments\\\":{},\\\"inputs\\\":{},\\\"outputs\\\":{},\\\"metadata\\\":{},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n namespace: default\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-batch-process\\\\\\\"\\\\n uid: \\\\\\\"b855a708-e036-4bcf-9ec8-354916f8fd1c\\\\\\\"\\\\nspec:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n predictors:\\\\n - graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 10\\\\n \\\\n\\\"}}\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-06-03T17:46:15Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-06-03T17:46:15Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-06-03T17:46:16Z\" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-06-03T17:46:16Z\" level=info msg=\"No output parameters\"\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tsleep: missing operand\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tTry 'sleep --help' for more information.\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 0 of 10 updated replicas are available...\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 1 of 10 updated replicas are available...\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 2 of 10 updated replicas are available...\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 3 of 10 updated replicas are available...\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 4 of 10 updated replicas are available...\r\n", "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 5 of 10 updated replicas are available...\r\n", "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 6 of 10 updated replicas are available...\r\n", "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 7 of 10 updated replicas are available...\r\n", @@ -870,10 +875,5631 @@ "\u001b[32mwait-seldon-resource\u001b[0m:\tdeployment \"sklearn-default-0-classifier\" successfully rolled out\r\n", "\u001b[34mdownload-object-store\u001b[0m:\tAdded `minio-local` successfully.\r\n", "\u001b[34mdownload-object-store\u001b[0m:\t`minio-local/data/input-data.txt` -> `/assets/input-data.txt`\r\n", - "\u001b[34mdownload-object-store\u001b[0m:\tTotal: 0 B, Transferred: 614.15 KiB, Speed: 61.21 MiB/s\r\n", - "\u001b[31mupload-object-store\u001b[0m:\tAdded `minio-local` successfully.\r\n", - "\u001b[31mupload-object-store\u001b[0m:\t`/assets/output-data.txt` -> `minio-local/data/output-data-72ad4430-c70a-4976-8043-2ba6c2f56772.txt`\r\n", - "\u001b[31mupload-object-store\u001b[0m:\tTotal: 0 B, Transferred: 1.31 MiB, Speed: 25.94 MiB/s\r\n" + "\u001b[34mdownload-object-store\u001b[0m:\tTotal: 0 B, Transferred: 614.15 KiB, Speed: 57.23 MiB/s\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-35:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-22:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-4:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-89:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-19:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-7:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-27:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-17:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-32:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-24:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-16:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-26:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-33:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-10:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-31:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-15:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-5:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-6:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-18:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-11:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-38:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-34:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-29:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-45:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-23:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-40:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-25:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-9:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-8:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-28:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-39:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-36:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-37:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-12:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-47:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-48:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-43:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-46:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-44:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-14:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-42:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-41:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-50:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-53:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-52:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-51:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-57:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-56:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-49:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-67:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-60:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-72:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-77:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-66:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-59:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-71:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-70:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-80:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-73:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-82:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-76:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-86:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-69:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-13:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-78:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-81:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-20:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-30:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-54:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-21:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-63:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-61:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-58:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-55:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-65:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-75:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-68:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-74:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-64:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-83:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-85:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-87:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-79:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-1:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-90:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-84:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-62:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-3:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-2:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-88:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-91:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-95:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-94:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-92:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-93:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-96:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-99:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-97:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-100:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tException in thread Thread-98:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 159, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t (self._dns_host, self.port), self.timeout, **extra_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 80, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise err\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py\", line 70, in create_connection\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t sock.connect(sa)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tConnectionRefusedError: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 600, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t chunked=chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 354, in _make_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn.request(method, url, **httplib_request_kw)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1244, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_request(method, url, body, headers, encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1290, in _send_request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.endheaders(body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1239, in endheaders\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._send_output(message_body, encode_chunked=encode_chunked)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 1026, in _send_output\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.send(msg)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/http/client.py\", line 966, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.connect()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 181, in connect\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t conn = self._new_conn()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connection.py\", line 168, in _new_conn\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self, \"Failed to establish a new connection: %s\" % e)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.NewConnectionError: : Failed to establish a new connection: [Errno 111] Connection refused\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 449, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t timeout=timeout\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py\", line 638, in urlopen\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t _stacktrace=sys.exc_info()[2])\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py\", line 399, in increment\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise MaxRetryError(_pool, url, error or ResponseError(cause))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\turllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tDuring handling of the above exception, another exception occurred:\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\tTraceback (most recent call last):\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self.run()\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/threading.py\", line 870, in run\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t self._target(*self._args, **self._kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/microservice/python/seldon_core/batch_processor.py\", line 144, in _start_request_worker\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t response = session.post(url, data=line, headers=headers)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 581, in post\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t return self.request('POST', url, data=data, json=json, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 533, in request\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t resp = self.send(prep, **send_kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/sessions.py\", line 646, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t r = adapter.send(request, **kwargs)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t File \"/opt/conda/lib/python3.7/site-packages/requests/adapters.py\", line 516, in send\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\t raise ConnectionError(e, request=request)\r\n", + "\u001b[39mprocess-batch-inputs\u001b[0m:\trequests.exceptions.ConnectionError: HTTPConnectionPool(host='istio-ingressgateway.istio-system.svc.cluster.local', port=80): Max retries exceeded with url: /seldon/default/sklearn/api/v1.0/predictions (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))\r\n" ] } ], @@ -967,7 +6593,7 @@ }, { "cell_type": "code", - "execution_count": 778, + "execution_count": 47, "metadata": {}, "outputs": [ { @@ -981,6 +6607,13 @@ "source": [ "!argo delete seldon-batch-process" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/examples/batch/argo-workflows/README_saved_from_latest.ipynb b/examples/batch/argo-workflows/README_saved_from_latest.ipynb new file mode 100644 index 0000000000..99019a88d0 --- /dev/null +++ b/examples/batch/argo-workflows/README_saved_from_latest.ipynb @@ -0,0 +1,459 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batch processing with Argo Worfklows\n", + "\n", + "In this notebook we will dive into how you can run batch processing with Argo Workflows and Seldon Core.\n", + "\n", + "Dependencies:\n", + "\n", + "* Seldon core installed as per the docs with an ingress\n", + "* Argo Workfklows installed in cluster (and argo CLI for commands)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Seldon Core Batch with Object Store\n", + "\n", + "In some cases we may want to read the data from an object source.\n", + "\n", + "In this case we will show how you can read from an object store, in this case minio.\n", + "\n", + "The workflow will look as follows:\n", + "\n", + "![](assets/seldon-batch.jpg)\n", + "\n", + "For this we will assume you have installed the Minio (mc) CLI - we will use a Minio client in the cluster but you can use another object store provider like S3, Google Cloud, Azure, etc.\n", + "\n", + "### Set up Minio in your cluster" + ] + }, + { + "cell_type": "code", + "execution_count": 481, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: minio\n", + "LAST DEPLOYED: Thu Apr 30 10:57:00 2020\n", + "NAMESPACE: default\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n", + "NOTES:\n", + "Minio can be accessed via port 9000 on the following DNS name from within your cluster:\n", + "minio.default.svc.cluster.local\n", + "\n", + "To access Minio from localhost, run the below commands:\n", + "\n", + " 1. export POD_NAME=$(kubectl get pods --namespace default -l \"release=minio\" -o jsonpath=\"{.items[0].metadata.name}\")\n", + "\n", + " 2. kubectl port-forward $POD_NAME 9000 --namespace default\n", + "\n", + "Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/\n", + "\n", + "You can now access Minio server on http://localhost:9000. Follow the below steps to connect to Minio server with mc client:\n", + "\n", + " 1. Download the Minio mc client - https://docs.minio.io/docs/minio-client-quickstart-guide\n", + "\n", + " 2. mc config host add minio-local http://localhost:9000 minioadmin minioadmin S3v4\n", + "\n", + " 3. mc ls minio-local\n", + "\n", + "Alternately, you can use your browser or the Minio SDK to access the server - https://docs.minio.io/categories/17\n" + ] + } + ], + "source": [ + "%%bash \n", + "helm install minio stable/minio \\\n", + " --set accessKey=minioadmin \\\n", + " --set secretKey=minioadmin \\\n", + " --set image.tag=RELEASE.2020-04-15T19-42-18Z" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Forward the Minio port so you can access it\n", + "\n", + "You can do this by runnning the following command in your terminal:\n", + "```\n", + "kubectl port-forward svc/minio 9000:9000\n", + " ```\n", + " \n", + "### Configure local minio client" + ] + }, + { + "cell_type": "code", + "execution_count": 483, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[m\u001b[32mAdded `minio-local` successfully.\u001b[0m\r\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!mc config host add minio-local http://localhost:9000 minioadmin minioadmin" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create some input for our model\n", + "\n", + "We will create a file that will contain 10,000 inputs that will be sent to our model" + ] + }, + { + "cell_type": "code", + "execution_count": 484, + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"assets/input-data.txt\", \"w\") as f:\n", + " for i in range(10000):\n", + " f.write('{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"' + str(i) + '\"}}\\n')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Check the contents of the file" + ] + }, + { + "cell_type": "code", + "execution_count": 485, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10000 assets/input-data.txt\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"0\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"1\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"2\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"3\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"4\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"5\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"6\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"7\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"8\"}}\n", + "{\"data\":{\"ndarray\":[[1, 2, 3, 4]]}, \"meta\": { \"puid\": \"9\"}}\n" + ] + } + ], + "source": [ + "!wc -l assets/input-data.txt\n", + "!head assets/input-data.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Upload the file to our minio" + ] + }, + { + "cell_type": "code", + "execution_count": 486, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[m\u001b[32;1mBucket created successfully `minio-local/data`.\u001b[0m\n", + "...-data.txt: 614.15 KiB / 614.15 KiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 9.74 MiB/s 0s\u001b[0m\u001b[0m\u001b[m\u001b[32;1m" + ] + } + ], + "source": [ + "!mc mb minio-local/data\n", + "!mc cp assets/input-data.txt minio-local/data/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create Argo Workflow\n", + "\n", + "In order to create our argo workflow we have made it simple so you can leverage the power of the helm charts.\n", + "\n", + "Before we dive into the contents of the full helm chart, let's first give it a try with some of the settings.\n", + "\n", + "We will run a batch job that will set up a Seldon Deployment with 10 replicas and 100 batch client workers to send requests." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldon-batch-workflow\r\n" + ] + } + ], + "source": [ + "!ls helm-charts/" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: seldon-batch-process\r\n", + "Namespace: default\r\n", + "ServiceAccount: default\r\n", + "Status: Pending\r\n", + "Created: Mon May 18 21:55:31 +0100 (now)\r\n" + ] + } + ], + "source": [ + "!helm template seldon-batch-workflow helm-charts/seldon-batch-workflow \\\n", + " --set workflow.name=seldon-batch-process \\\n", + " --set seldonDeployment.create=false \\\n", + " --set seldonDeployment.name=\"seldon-batch\" \\\n", + " --set seldonDeployment.replicas=10 \\\n", + " --set batchWorker.workers=100 \\\n", + " --set minio.inputDataPath=\"s3://data/input-data.txt\" \\\n", + " --set minio.outputDataPath=\"s3://data/output-data-{{workflow.uid}}.txt\" \\\n", + " | argo submit -" + ] + }, + { + "cell_type": "code", + "execution_count": 120, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE DURATION PRIORITY\r\n", + "seldon-batch-process Running 46s 46s 0\r\n" + ] + } + ], + "source": [ + "!argo list" + ] + }, + { + "cell_type": "code", + "execution_count": 121, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: seldon-batch-process\r\n", + "Namespace: default\r\n", + "ServiceAccount: default\r\n", + "Status: Running\r\n", + "Created: Mon May 18 21:55:31 +0100 (53 seconds ago)\r\n", + "Started: Mon May 18 21:55:31 +0100 (53 seconds ago)\r\n", + "Duration: 53 seconds\r\n", + "\r\n", + "\u001b[39mSTEP\u001b[0m PODNAME DURATION MESSAGE\r\n", + " \u001b[36m●\u001b[0m seldon-batch-process (seldon-batch-process) \r\n", + " ├---\u001b[32m✔\u001b[0m create-seldon-resource (create-seldon-resource-template) seldon-batch-process-3626514072 1s \r\n", + " ├---\u001b[32m✔\u001b[0m wait-seldon-resource (wait-seldon-resource-template) seldon-batch-process-2052519094 30s \r\n", + " └---\u001b[36m●\u001b[0m process-batch-inputs (process-batch-inputs-template) seldon-batch-process-50851621 20s \r\n" + ] + } + ], + "source": [ + "!argo get seldon-batch-process" + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-18T20:52:21Z\" level=info msg=\"Starting Workflow Executor\" version=vv2.7.4+50b209c.dirty\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-18T20:52:21Z\" level=info msg=\"Creating a docker executor\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-18T20:52:21Z\" level=info msg=\"Executor (version: vv2.7.4+50b209c.dirty, build_date: 2020-04-16T16:37:57Z) initialized (pod: default/seldon-batch-process-3626514072) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"arguments\\\":{},\\\"inputs\\\":{},\\\"outputs\\\":{},\\\"metadata\\\":{},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"seldon-844299e9-abf7-4742-9d4d-eeb405433569\\\\\\\"\\\\n namespace: default\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-batch-process\\\\\\\"\\\\n uid: \\\\\\\"844299e9-abf7-4742-9d4d-eeb405433569\\\\\\\"\\\\nspec:\\\\n name: \\\\\\\"seldon-844299e9-abf7-4742-9d4d-eeb405433569\\\\\\\"\\\\n predictors:\\\\n - graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 1\\\\n \\\\n\\\"}}\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-18T20:52:21Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-18T20:52:21Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-18T20:52:22Z\" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/seldon-844299e9-abf7-4742-9d4d-eeb405433569\r\n", + "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-05-18T20:52:22Z\" level=info msg=\"No output parameters\"\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"seldon-70a32d330b04187c79768e37cda5d600\" rollout to finish: 0 of 1 updated replicas are available...\r\n", + "\u001b[32mwait-seldon-resource\u001b[0m:\tdeployment \"seldon-70a32d330b04187c79768e37cda5d600\" successfully rolled out\r\n" + ] + } + ], + "source": [ + "!argo logs -w seldon-batch-process " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Check output in object store\n", + "\n", + "We can now visualise the output that we obtained in the object store.\n", + "\n", + "First we can check that the file is present:" + ] + }, + { + "cell_type": "code", + "execution_count": 114, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Workflow ID is 844299e9-abf7-4742-9d4d-eeb405433569\n" + ] + } + ], + "source": [ + "import json\n", + "wf_arr = !argo get seldon-batch-process -o json\n", + "wf = json.loads(\"\".join(wf_arr))\n", + "WF_ID = wf[\"metadata\"][\"uid\"]\n", + "print(f\"Workflow ID is {WF_ID}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 115, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[m\u001b[32m[2020-05-18 21:53:16 BST] \u001b[0m\u001b[33m 1.3MiB \u001b[0m\u001b[1moutput-data-844299e9-abf7-4742-9d4d-eeb405433569.txt\u001b[0m\r\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!mc ls minio-local/data/output-data-\"$WF_ID\".txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we can output the contents of the file created using the `mc head` command." + ] + }, + { + "cell_type": "code", + "execution_count": 116, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "...33569.txt: 1.31 MiB / 1.31 MiB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 26.12 MiB/s 0s\u001b[0m\u001b[0m\u001b[m\u001b[32;1m{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"4\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"0\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"3\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"1\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"100\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"6\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"7\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"9\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"8\"}}\n", + "{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.0006985194531162841,0.003668039039435755,0.9956334415074478]]},\"meta\":{\"puid\":\"106\"}}\n" + ] + } + ], + "source": [ + "!mc cp minio-local/data/output-data-\"$WF_ID\".txt assets/output-data.txt\n", + "!head assets/output-data.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 122, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Workflow 'seldon-batch-process' deleted\r\n" + ] + } + ], + "source": [ + "!argo delete seldon-batch-process" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/Chart.yaml b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/Chart.yaml new file mode 100644 index 0000000000..bf39237512 --- /dev/null +++ b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +description: Seldon Batch Workflow +keywords: +- kubernetes +- machine-learning +name: seldon-batch-workflow +sources: +- https://github.com/SeldonIO/seldon-core +version: 0.1 diff --git a/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/README.md b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/README.md new file mode 100644 index 0000000000..47e2829975 --- /dev/null +++ b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/README.md @@ -0,0 +1,3 @@ +# Seldon Batch Workflow + +This chart creates a batch workflow which leverages the seldon batch processor functionality. diff --git a/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/templates/workflow.yaml b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/templates/workflow.yaml new file mode 100644 index 0000000000..cca77bcde9 --- /dev/null +++ b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/templates/workflow.yaml @@ -0,0 +1,88 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + {{- if eq .Values.workflow.useNameAsGenerateName false }} + name: {{ .Values.workflow.name }} + {{- else }} + generateName: {{ .Values.workflow.name }} + {{- end }} + namespace: {{ .Values.workflow.namespace }} +spec: + entrypoint: seldon-batch-process + templates: + - name: seldon-batch-process + steps: + {{- if eq .Values.seldonDeployment.create true }} + - - name: create-seldon-resource + template: create-seldon-resource-template + - - name: wait-seldon-resource + template: wait-seldon-resource-template + {{- end }} + - - name: process-batch-inputs + template: process-batch-inputs-template + + {{- if eq .Values.seldonDeployment.create true }} + - name: create-seldon-resource-template + resource: + action: create + manifest: | + apiVersion: machinelearning.seldon.io/v1 + kind: SeldonDeployment + metadata: + name: "{{ .Values.seldonDeployment.name }}" + namespace: {{ .Values.workflow.namespace }} + ownerReferences: + - apiVersion: argoproj.io/v1alpha1 + blockOwnerDeletion: true + kind: Workflow + name: "{{`{{workflow.name}}`}}" + uid: "{{`{{workflow.uid}}`}}" + spec: + name: "{{ .Values.seldonDeployment.name }}" + predictors: + - graph: + children: [] + implementation: {{ .Values.seldonDeployment.server }} + modelUri: {{ .Values.seldonDeployment.modelUri }} + name: classifier + name: default + replicas: {{ .Values.seldonDeployment.replicas }} + + - name: wait-seldon-resource-template + script: + image: bitnami/kubectl:1.17 + command: [bash] + source: | + sleep 3 + kubectl rollout status \ + deploy/$(kubectl get deploy -l seldon-deployment-id="{{ .Values.seldonDeployment.name }}" -o jsonpath='{.items[0].metadata.name}') + {{- end }} + + - name: process-batch-inputs-template + script: + image: {{ .Values.seldonDeployment.image }} + command: [bash] + source: | + seldon-batch-processor \ + --deployment-name "{{ .Values.seldonDeployment.name }}" \ + --namespace "{{ .Values.workflow.namespace }}" \ + --host "{{ .Values.batchWorker.host }}" \ + --workers "{{ .Values.batchWorker.workers }}" \ + --retries "{{ .Values.batchWorker.retries }}" \ + --input-data-path "{{ .Values.minio.inputDataPath }}" \ + --output-data-path "{{ .Values.minio.outputDataPath }}" + env: + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: {{ .Values.minio.secret.name }} + key: {{ .Values.minio.secret.keyName.accesskey }} + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.minio.secret.name }} + key: {{ .Values.minio.secret.keyName.secretkey }} + - name: AWS_ENDPOINT_URL + value: {{ .Values.minio.endpoint }} + diff --git a/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/values.yaml b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/values.yaml new file mode 100644 index 0000000000..47e23806ae --- /dev/null +++ b/examples/batch/argo-workflows/helm-charts-saved-from-latest/seldon-batch-workflow/values.yaml @@ -0,0 +1,45 @@ +workflow: + # Name of the overarching argo workflow + name: seldon-batch-process + # If true the randomly generated string will be apended as the name of the workflow + useNameAsGenerateName: false + # Namespace where to create the workflow and all resources in batch job + namespace: default +# Seldon deployment to be created for batch processing +seldonDeployment: + # Whether to create the deployment or to skip creation + create: true + # Name to use for the seldon deployment which by default appends generated workflow ID + name: seldon-{{workflow.uid}} + # Image to use for the batch client + image: seldonio/seldon-core-s2i-python37:1.1.1-SNAPSHOT + # Prepackaged model server to use [see https://docs.seldon.io/projects/seldon-core/en/latest/servers/overview.html] + server: SKLEARN_SERVER + # The URL for the model that is to be used + modelUri: gs://seldon-models/sklearn/iris + # The number of seldon deployment replicas to launch + replicas: 10 +# The batch worker is the component that will send the requests from the files +batchWorker: + # Endpoint of for the batch client to contact the seldon deployment + host: istio-ingressgateway.istio-system.svc.cluster.local + # Number of parallel batch client workers to process the data + workers: 100 + # Number of times client will try to send a request if it fails + retries: 3 +minio: + # The location of the minio endpoint + endpoint: http://minio.default.svc.cluster.local:9000 + # This is the secret that should contain the values to access minio + secret: + # The name of the secret which by default is "minio" but you can create a different one + name: minio + keyName: + # The key name inside that secret to find the access key to authenticate minio + accesskey: accesskey + # The key name inside that secret to find the secret key to authenticate minio + secretkey: secretkey + # The name of the file inside of minio that will contain the batch data to process + inputDataPath: data/input-data.txt + # The name of the file inside of minio that will contain the batch data to process + outputDataPath: data/output-data-{{workflow.uid}}.txt diff --git a/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/templates/workflow.yaml b/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/templates/workflow.yaml index c8fe42502c..528608d173 100644 --- a/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/templates/workflow.yaml +++ b/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/templates/workflow.yaml @@ -10,6 +10,20 @@ metadata: namespace: {{ .Values.workflow.namespace }} spec: entrypoint: seldon-batch-process + volumeClaimTemplates: + - metadata: + name: "{{ .Values.pvc.name }}" + ownerReferences: + - apiVersion: argoproj.io/v1alpha1 + blockOwnerDeletion: true + kind: Workflow + name: "{{`{{workflow.name}}`}}" + uid: "{{`{{workflow.uid}}`}}" + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: "{{ .Values.pvc.storage }}" templates: - name: seldon-batch-process steps: @@ -21,16 +35,8 @@ spec: template: download-object-store-template - - name: process-batch-inputs template: process-batch-inputs-template - arguments: - artifacts: - - name: input-data - from: "{{`{{steps.download-object-store.outputs.artifacts.input-data}}`}}" - - name: upload-object-store template: upload-object-store-template - arguments: - artifacts: - - name: output-data - from: "{{`{{steps.process-batch-inputs.outputs.artifacts.output-data}}`}}" - name: create-seldon-resource-template resource: @@ -63,7 +69,7 @@ spec: image: bitnami/kubectl:1.17 command: [bash] source: | - sleep 3 + sleep {{ .Values.seldonDeployment.waitTime }} kubectl rollout status \ deploy/$(kubectl get deploy -l seldon-deployment-id="{{ .Values.seldonDeployment.name }}" -o jsonpath='{.items[0].metadata.name}') @@ -83,26 +89,20 @@ spec: key: {{ .Values.minio.secret.keyName.secretkey }} - name: MINIO_SERVER_HOST value: {{ .Values.minio.endpoint }} + volumeMounts: + - name: "{{ .Values.pvc.name }}" + mountPath: /assets command: [sh] source: | mc config host add minio-local $MINIO_SERVER_HOST $MINIO_SERVER_ACCESS_KEY $MINIO_SERVER_ACCESS_KEY - mc cp minio-local/data/input-data.txt /assets/input-data.txt - outputs: - artifacts: - - name: input-data - path: /assets/input-data.txt + mc cp minio-local/{{ .Values.minio.inputDataPath }} /assets/input-data.txt - name: process-batch-inputs-template - inputs: - artifacts: - - name: input-data - path: /assets/input-data.txt - outputs: - artifacts: - - name: output-data - path: /assets/output-data.txt script: image: {{ .Values.seldonDeployment.image }} + volumeMounts: + - name: "{{ .Values.pvc.name }}" + mountPath: /assets command: [bash] source: | seldon-batch-processor \ @@ -115,12 +115,11 @@ spec: --output-data-path "/assets/output-data.txt" - name: upload-object-store-template - inputs: - artifacts: - - name: output-data - path: /assets/output-data.txt script: image: minio/mc:RELEASE.2020-04-17T08-55-48Z + volumeMounts: + - name: "{{ .Values.pvc.name }}" + mountPath: /assets command: [sh] env: - name: MINIO_SERVER_ACCESS_KEY diff --git a/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/values.yaml b/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/values.yaml index c9506f48ff..3745921f94 100644 --- a/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/values.yaml +++ b/examples/batch/argo-workflows/helm-charts/seldon-batch-workflow/values.yaml @@ -5,6 +5,11 @@ workflow: useNameAsGenerateName: false # Namespace where to create the workflow and all resources in batch job namespace: default +pvc: + # Name of the persistent volume claim to be created + name: seldon-pvc + # Size of the storage volume to be created for the batch job + storage: 2Mi # Seldon deployment to be created for batch processing seldonDeployment: # Name to use for the seldon deployment which by default appends generated workflow ID @@ -25,6 +30,8 @@ batchWorker: workers: 100 # Number of times client will try to send a request if it fails retries: 3 + # Waiting time before checks for deployment to ensure kubernetes cluster registers create + waitTime: 5 minio: # The location of the minio endpoint endpoint: http://minio.default.svc.cluster.local:9000