From 18e3c335a22745831f445f30d05eee34c2f98465 Mon Sep 17 00:00:00 2001 From: Alejandro Saucedo Date: Sat, 27 Jun 2020 11:10:35 +0100 Subject: [PATCH] Added notebook instructtions on patch fix --- .../upgrade_1_2_volume_patch/README.ipynb | 810 ++++++++++++++++++ .../upgrade_1_2_volume_patch/README.md | 558 ++++++++++++ .../patch_volumes_1_2.py | 51 ++ 3 files changed, 1419 insertions(+) create mode 100644 examples/infrastructure/upgrade_1_2_volume_patch/README.ipynb create mode 100644 examples/infrastructure/upgrade_1_2_volume_patch/README.md create mode 100644 examples/infrastructure/upgrade_1_2_volume_patch/patch_volumes_1_2.py diff --git a/examples/infrastructure/upgrade_1_2_volume_patch/README.ipynb b/examples/infrastructure/upgrade_1_2_volume_patch/README.ipynb new file mode 100644 index 0000000000..4034c68bd8 --- /dev/null +++ b/examples/infrastructure/upgrade_1_2_volume_patch/README.ipynb @@ -0,0 +1,810 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Upgrading from 1.1 to 1.2 Volume Patch\n", + "\n", + "This notebook contains an overview of how to perform the patch when upgrading from Seldon Core 1.1 into 1.2.\n", + "\n", + "Note that this is ONLY required if you are performing a rolling upgrade. If you can delete the previous version and install Seldon Core 1.2 you will not need to perform any patching.\n", + "\n", + "This issue will be fixed in version 1.2.1, so it is recommended to upgrade to this version instead.\n", + "\n", + "In this notebook we will:\n", + "* Install Seldon Core version 1.1\n", + "* Deploy 3 models with varying complexities and specifications\n", + "* Perform upgrade\n", + "* Observe Issues\n", + "* Run patch\n", + "* Confirm issues are resolved" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install Seldon Core Version 1.1" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Release \"seldon-core\" does not exist. Installing it now.\n", + "NAME: seldon-core\n", + "LAST DEPLOYED: Sat Jun 27 10:52:41 2020\n", + "NAMESPACE: seldon-system\n", + "STATUS: deployed\n", + "REVISION: 1\n", + "TEST SUITE: None\n" + ] + } + ], + "source": [ + "%%bash\n", + "kubectl create namespace seldon-system || echo \"Namespace seldon-system already exists\"\n", + "helm upgrade --install seldon-core seldon-core-operator \\\n", + " --repo https://storage.googleapis.com/seldon-charts \\\n", + " --namespace seldon-system \\\n", + " --version v1.1.0 \\\n", + " --set certManager.enabled=\"true\" \\\n", + " --set usageMetrics.enabled=true \\\n", + " --set istio.enabled=\"true\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check seldon controller manager is running correctly" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldon-controller-manager-6978f54b99-xvgvd 1/1 Running 0 7m28s\r\n" + ] + } + ], + "source": [ + "!kubectl get pods -n seldon-system | grep seldon-controller" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Check no errors in logs" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2020-06-27T09:59:37.767Z\tDEBUG\tcontroller-runtime.controller\tSuccessfully Reconciled\t{\"controller\": \"seldon-controller-manager\", \"request\": \"seldon-system/sklearn\"}\r\n", + "2020-06-27T09:59:37.767Z\tDEBUG\tcontroller-runtime.manager.events\tNormal\t{\"object\": {\"kind\":\"SeldonDeployment\",\"namespace\":\"seldon-system\",\"name\":\"sklearn\",\"uid\":\"4fca069c-eab1-4903-ad23-40517c91207b\",\"apiVersion\":\"machinelearning.seldon.io/v1\",\"resourceVersion\":\"1718083\"}, \"reason\": \"Updated\", \"message\": \"Updated SeldonDeployment \\\"sklearn\\\"\"}\r\n" + ] + } + ], + "source": [ + "!kubectl logs -n seldon-system -l control-plane=seldon-controller-manager | tail -2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Deploy 3 models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### First model is simple sklearn model in default namespace" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/sklearn created\n" + ] + } + ], + "source": [ + "%%bash\n", + "kubectl apply -n default -f - << END\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: sklearn\n", + "spec:\n", + " name: iris\n", + " predictors:\n", + " - graph:\n", + " children: []\n", + " implementation: SKLEARN_SERVER\n", + " modelUri: gs://seldon-models/sklearn/iris\n", + " name: classifier\n", + " name: default\n", + " replicas: 1\n", + " svcOrchSpec:\n", + " env:\n", + " - name: SELDON_LOG_LEVEL\n", + " value: DEBUG\n", + "END" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Second model is the same sklaern model but in the seldon-system namespace" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/sklearn created\n" + ] + } + ], + "source": [ + "%%bash\n", + "kubectl apply -n seldon-system -f - << END\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: sklearn\n", + "spec:\n", + " name: iris\n", + " predictors:\n", + " - graph:\n", + " children: []\n", + " implementation: SKLEARN_SERVER\n", + " modelUri: gs://seldon-models/sklearn/iris\n", + " name: classifier\n", + " name: default\n", + " replicas: 1\n", + " svcOrchSpec:\n", + " env:\n", + " - name: SELDON_LOG_LEVEL\n", + " value: DEBUG\n", + "END" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Third model is the iris custom model with a mounted volume from a secret" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we create the secret" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "secret/seldon-test-secret created\n" + ] + } + ], + "source": [ + "%%bash \n", + "kubectl create secret generic seldon-test-secret --from-literal=file1.txt=contents --from-literal=file2.txt=morecontents" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we deploy the model" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/seldon-deployment-example created\n" + ] + } + ], + "source": [ + "%%bash\n", + "kubectl apply -f - << END\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: seldon-deployment-example\n", + "spec:\n", + " name: sklearn-iris-deployment\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " volumes:\n", + " - name: \"secret-mount\"\n", + " volumeSource:\n", + " secret: \"seldon-test-secret\"\n", + " containers:\n", + " - image: seldonio/sklearn-iris:0.1\n", + " imagePullPolicy: IfNotPresent\n", + " name: sklearn-iris-classifier\n", + " volumeMounts:\n", + " - name: \"secret-mount\"\n", + " mountPath: \"/cert/\"\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: sklearn-iris-classifier\n", + " type: MODEL\n", + " name: sklearn-iris-predictor\n", + " replicas: 1\n", + "END" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we wait until they are deployed" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAMESPACE NAME AGE\r\n", + "default seldon-deployment-example 39s\r\n", + "default sklearn 60s\r\n", + "seldon-system sklearn 55s\r\n" + ] + } + ], + "source": [ + "!kubectl get sdep --all-namespaces" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "seldon-92a927e5e90d7602e08ba9b9304f70e8-8544bc96d-qkm6x 2/2 Running 0 73s\n", + "sklearn-default-0-classifier-777f84985b-9tj5r 2/2 Running 0 94s\n", + "NAME READY STATUS RESTARTS AGE\n", + "seldon-controller-manager-6978f54b99-xvgvd 1/1 Running 0 6m57s\n", + "sklearn-default-0-classifier-748c59789b-2lnvh 2/2 Running 0 89s\n" + ] + } + ], + "source": [ + "!kubectl get pods -n default && kubectl get pods -n seldon-system " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Perform upgrade to 1.2" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Namespace seldon-system already exists\n", + "Release \"seldon-core\" has been upgraded. Happy Helming!\n", + "NAME: seldon-core\n", + "LAST DEPLOYED: Sat Jun 27 11:03:18 2020\n", + "NAMESPACE: seldon-system\n", + "STATUS: deployed\n", + "REVISION: 2\n", + "TEST SUITE: None\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon-system\" already exists\n" + ] + } + ], + "source": [ + "%%bash\n", + "helm upgrade --install seldon-core seldon-core-operator \\\n", + " --repo https://storage.googleapis.com/seldon-charts \\\n", + " --namespace seldon-system \\\n", + " --version v1.2.0 \\\n", + " --set certManager.enabled=\"true\" \\\n", + " --set usageMetrics.enabled=true \\\n", + " --set istio.enabled=\"true\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Observe error" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "k8s.io/apimachinery/pkg/util/wait.JitterUntil\r\n", + "\t/go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153\r\n", + "k8s.io/apimachinery/pkg/util/wait.Until\r\n", + "\t/go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88\r\n", + "2020-06-27T10:04:01.898Z\tDEBUG\tcontroller-runtime.manager.events\tWarning\t{\"object\": {\"kind\":\"SeldonDeployment\",\"namespace\":\"seldon-system\",\"name\":\"sklearn\",\"uid\":\"4fca069c-eab1-4903-ad23-40517c91207b\",\"apiVersion\":\"machinelearning.seldon.io/v1\",\"resourceVersion\":\"1719032\"}, \"reason\": \"InternalError\", \"message\": \"Deployment.apps \\\"sklearn-default-0-classifier\\\" is invalid: [spec.template.spec.containers[0].volumeMounts[0].name: Not found: \\\"podinfo\\\", spec.template.spec.containers[0].volumeMounts[1].mountPath: Invalid value: \\\"/etc/podinfo\\\": must be unique]\"}\r\n" + ] + } + ], + "source": [ + "!kubectl logs -n seldon-system -l control-plane=seldon-controller-manager | tail -5" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run Patch" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The error is due a rename on the volumeMounts. We have created the script below which goes through all the seldon deploymetns across all namespaces to rename the volumeMount from podinfo to \"seldon-podinfo\".\n", + "\n", + "It is recommended to understand this script fully if this is to be run in prodution as it woudl clash if any existing volume is actually named \"podinfo\"." + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting patch_volumes_1_2.py\n" + ] + } + ], + "source": [ + "%%writefile patch_volumes_1_2.py\n", + "#!/usr/bin/env python3\n", + "\n", + "import yaml\n", + "import subprocess\n", + "import os\n", + "import time\n", + "\n", + "def run(cmd: str):\n", + " cmd_arr = cmd.split()\n", + " output = subprocess.Popen(cmd_arr,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.STDOUT).communicate()\n", + " output_str = [out.decode() for out in output if out]\n", + " return \"\\n\".join(output_str)\n", + "\n", + "def patch_volumes_seldon_1_2():\n", + "\n", + " namespaces = run(\"kubectl get ns -o=name\")\n", + "\n", + " for namespace in namespaces.split():\n", + " namespace = namespace.replace(\"namespace/\", \"\")\n", + " sdeps_raw = run(f\"kubectl get sdep -o yaml -n {namespace}\")\n", + " sdeps_dict = yaml.safe_load(sdeps_raw)\n", + " sdep_list = sdeps_dict.get(\"items\")\n", + " if sdep_list:\n", + " for sdep in sdep_list:\n", + " name = sdep.get(\"metadata\", {}).get(\"name\")\n", + " print(f\"Processing {name} in namespace {namespace}\")\n", + " predictors = sdep.get(\"spec\", {}).get(\"predictors\", [])\n", + " for predictor in predictors:\n", + " for component_spec in predictor.get(\"componentSpecs\", []):\n", + " for container in component_spec.get(\"spec\", {}).get(\"containers\", []):\n", + " for volume_mount in container.get(\"volumeMounts\", []):\n", + " if volume_mount.get(\"name\") == \"podinfo\":\n", + " print(\"Patching volume\")\n", + " volume_mount[\"name\"] = \"seldon-podinfo\"\n", + "\n", + " with open(\"seldon_tmp.yaml\", \"w\") as tmp_file:\n", + " yaml.dump(sdep, tmp_file)\n", + " run(\"kubectl apply -f seldon_tmp.yaml\")\n", + "\n", + " print(yaml.dump(sdep))\n", + " os.remove(\"seldon_tmp.yaml\")\n", + "\n", + "\n", + "if __name__ == \"__main__\":\n", + " patch_volumes_seldon_1_2()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run script" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing seldon-deployment-example in namespace default\n", + "Patching volume\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " annotations:\n", + " kubectl.kubernetes.io/last-applied-configuration: '{\"apiVersion\":\"machinelearning.seldon.io/v1\",\"kind\":\"SeldonDeployment\",\"metadata\":{\"annotations\":{},\"name\":\"seldon-deployment-example\",\"namespace\":\"default\"},\"spec\":{\"name\":\"sklearn-iris-deployment\",\"predictors\":[{\"componentSpecs\":[{\"spec\":{\"containers\":[{\"image\":\"seldonio/sklearn-iris:0.1\",\"imagePullPolicy\":\"IfNotPresent\",\"name\":\"sklearn-iris-classifier\",\"volumeMounts\":[{\"mountPath\":\"/cert/\",\"name\":\"secret-mount\"}]}],\"volumes\":[{\"name\":\"secret-mount\",\"volumeSource\":{\"secret\":\"seldon-test-secret\"}}]}}],\"graph\":{\"children\":[],\"endpoint\":{\"type\":\"REST\"},\"name\":\"sklearn-iris-classifier\",\"type\":\"MODEL\"},\"name\":\"sklearn-iris-predictor\",\"replicas\":1}]}}\n", + "\n", + " '\n", + " creationTimestamp: '2020-06-27T09:58:26Z'\n", + " generation: 1\n", + " name: seldon-deployment-example\n", + " namespace: default\n", + " resourceVersion: '1719036'\n", + " selfLink: /apis/machinelearning.seldon.io/v1/namespaces/default/seldondeployments/seldon-deployment-example\n", + " uid: 8a15eb91-e614-41d9-9d0e-abc191d3a417\n", + "spec:\n", + " name: sklearn-iris-deployment\n", + " predictors:\n", + " - componentSpecs:\n", + " - metadata:\n", + " creationTimestamp: null\n", + " spec:\n", + " containers:\n", + " - image: seldonio/sklearn-iris:0.1\n", + " imagePullPolicy: IfNotPresent\n", + " name: sklearn-iris-classifier\n", + " ports:\n", + " - containerPort: 6000\n", + " name: metrics\n", + " protocol: TCP\n", + " resources: {}\n", + " volumeMounts:\n", + " - mountPath: /cert/\n", + " name: secret-mount\n", + " - mountPath: /etc/podinfo\n", + " name: seldon-podinfo\n", + " volumes:\n", + " - name: secret-mount\n", + " engineResources: {}\n", + " graph:\n", + " endpoint:\n", + " service_host: localhost\n", + " service_port: 9000\n", + " type: REST\n", + " implementation: UNKNOWN_IMPLEMENTATION\n", + " name: sklearn-iris-classifier\n", + " type: MODEL\n", + " labels:\n", + " version: sklearn-iris-predictor\n", + " name: sklearn-iris-predictor\n", + " replicas: 1\n", + " svcOrchSpec: {}\n", + "status:\n", + " address:\n", + " url: http://seldon-deployment-example-sklearn-iris-predictor.default.svc.cluster.local:8000/api/v1.0/predictions\n", + " deploymentStatus:\n", + " seldon-92a927e5e90d7602e08ba9b9304f70e8:\n", + " availableReplicas: 1\n", + " replicas: 1\n", + " description: 'Deployment.apps \"seldon-92a927e5e90d7602e08ba9b9304f70e8\" is invalid:\n", + " [spec.template.spec.containers[0].volumeMounts[1].name: Not found: \"podinfo\",\n", + " spec.template.spec.containers[0].volumeMounts[2].mountPath: Invalid value: \"/etc/podinfo\":\n", + " must be unique]'\n", + " replicas: 1\n", + " serviceStatus:\n", + " seldon-d0934233541ef6b732c88680f8a0e94f:\n", + " httpEndpoint: seldon-d0934233541ef6b732c88680f8a0e94f.default:9000\n", + " svcName: seldon-d0934233541ef6b732c88680f8a0e94f\n", + " seldon-deployment-example-sklearn-iris-predictor:\n", + " grpcEndpoint: seldon-deployment-example-sklearn-iris-predictor.default:5001\n", + " httpEndpoint: seldon-deployment-example-sklearn-iris-predictor.default:8000\n", + " svcName: seldon-deployment-example-sklearn-iris-predictor\n", + " state: Failed\n", + "\n", + "Processing sklearn in namespace default\n", + "Patching volume\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " annotations:\n", + " kubectl.kubernetes.io/last-applied-configuration: '{\"apiVersion\":\"machinelearning.seldon.io/v1alpha2\",\"kind\":\"SeldonDeployment\",\"metadata\":{\"annotations\":{},\"name\":\"sklearn\",\"namespace\":\"default\"},\"spec\":{\"name\":\"iris\",\"predictors\":[{\"graph\":{\"children\":[],\"implementation\":\"SKLEARN_SERVER\",\"modelUri\":\"gs://seldon-models/sklearn/iris\",\"name\":\"classifier\"},\"name\":\"default\",\"replicas\":1,\"svcOrchSpec\":{\"env\":[{\"name\":\"SELDON_LOG_LEVEL\",\"value\":\"DEBUG\"}]}}]}}\n", + "\n", + " '\n", + " creationTimestamp: '2020-06-27T09:58:05Z'\n", + " generation: 1\n", + " name: sklearn\n", + " namespace: default\n", + " resourceVersion: '1719025'\n", + " selfLink: /apis/machinelearning.seldon.io/v1/namespaces/default/seldondeployments/sklearn\n", + " uid: 4f44a5dc-8da4-45ba-8ace-00e51643c7ff\n", + "spec:\n", + " name: iris\n", + " predictors:\n", + " - componentSpecs:\n", + " - metadata:\n", + " creationTimestamp: '2020-06-27T09:58:05Z'\n", + " spec:\n", + " containers:\n", + " - image: seldonio/sklearnserver_rest:0.3\n", + " name: classifier\n", + " ports:\n", + " - containerPort: 6000\n", + " name: metrics\n", + " protocol: TCP\n", + " resources: {}\n", + " volumeMounts:\n", + " - mountPath: /etc/podinfo\n", + " name: seldon-podinfo\n", + " engineResources: {}\n", + " graph:\n", + " endpoint:\n", + " service_host: localhost\n", + " service_port: 9000\n", + " type: REST\n", + " implementation: SKLEARN_SERVER\n", + " modelUri: gs://seldon-models/sklearn/iris\n", + " name: classifier\n", + " type: MODEL\n", + " labels:\n", + " version: default\n", + " name: default\n", + " replicas: 1\n", + " svcOrchSpec:\n", + " env:\n", + " - name: SELDON_LOG_LEVEL\n", + " value: DEBUG\n", + "status:\n", + " address:\n", + " url: http://sklearn-default.default.svc.cluster.local:8000/api/v1.0/predictions\n", + " deploymentStatus:\n", + " sklearn-default-0-classifier:\n", + " availableReplicas: 1\n", + " replicas: 1\n", + " description: 'Deployment.apps \"sklearn-default-0-classifier\" is invalid: [spec.template.spec.containers[0].volumeMounts[0].name:\n", + " Not found: \"podinfo\", spec.template.spec.containers[0].volumeMounts[1].mountPath:\n", + " Invalid value: \"/etc/podinfo\": must be unique]'\n", + " replicas: 1\n", + " serviceStatus:\n", + " sklearn-default:\n", + " grpcEndpoint: sklearn-default.default:5001\n", + " httpEndpoint: sklearn-default.default:8000\n", + " svcName: sklearn-default\n", + " sklearn-default-classifier:\n", + " httpEndpoint: sklearn-default-classifier.default:9000\n", + " svcName: sklearn-default-classifier\n", + " state: Failed\n", + "\n", + "Processing sklearn in namespace seldon-system\n", + "Patching volume\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " annotations:\n", + " kubectl.kubernetes.io/last-applied-configuration: '{\"apiVersion\":\"machinelearning.seldon.io/v1alpha2\",\"kind\":\"SeldonDeployment\",\"metadata\":{\"annotations\":{},\"name\":\"sklearn\",\"namespace\":\"seldon-system\"},\"spec\":{\"name\":\"iris\",\"predictors\":[{\"graph\":{\"children\":[],\"implementation\":\"SKLEARN_SERVER\",\"modelUri\":\"gs://seldon-models/sklearn/iris\",\"name\":\"classifier\"},\"name\":\"default\",\"replicas\":1,\"svcOrchSpec\":{\"env\":[{\"name\":\"SELDON_LOG_LEVEL\",\"value\":\"DEBUG\"}]}}]}}\n", + "\n", + " '\n", + " creationTimestamp: '2020-06-27T09:58:10Z'\n", + " generation: 1\n", + " name: sklearn\n", + " namespace: seldon-system\n", + " resourceVersion: '1719032'\n", + " selfLink: /apis/machinelearning.seldon.io/v1/namespaces/seldon-system/seldondeployments/sklearn\n", + " uid: 4fca069c-eab1-4903-ad23-40517c91207b\n", + "spec:\n", + " name: iris\n", + " predictors:\n", + " - componentSpecs:\n", + " - metadata:\n", + " creationTimestamp: '2020-06-27T09:58:10Z'\n", + " spec:\n", + " containers:\n", + " - image: seldonio/sklearnserver_rest:0.3\n", + " name: classifier\n", + " ports:\n", + " - containerPort: 6000\n", + " name: metrics\n", + " protocol: TCP\n", + " resources: {}\n", + " volumeMounts:\n", + " - mountPath: /etc/podinfo\n", + " name: seldon-podinfo\n", + " engineResources: {}\n", + " graph:\n", + " endpoint:\n", + " service_host: localhost\n", + " service_port: 9000\n", + " type: REST\n", + " implementation: SKLEARN_SERVER\n", + " modelUri: gs://seldon-models/sklearn/iris\n", + " name: classifier\n", + " type: MODEL\n", + " labels:\n", + " version: default\n", + " name: default\n", + " replicas: 1\n", + " svcOrchSpec:\n", + " env:\n", + " - name: SELDON_LOG_LEVEL\n", + " value: DEBUG\n", + "status:\n", + " address:\n", + " url: http://sklearn-default.seldon-system.svc.cluster.local:8000/api/v1.0/predictions\n", + " deploymentStatus:\n", + " sklearn-default-0-classifier:\n", + " availableReplicas: 1\n", + " replicas: 1\n", + " description: 'Deployment.apps \"sklearn-default-0-classifier\" is invalid: [spec.template.spec.containers[0].volumeMounts[0].name:\n", + " Not found: \"podinfo\", spec.template.spec.containers[0].volumeMounts[1].mountPath:\n", + " Invalid value: \"/etc/podinfo\": must be unique]'\n", + " replicas: 1\n", + " serviceStatus:\n", + " sklearn-default:\n", + " grpcEndpoint: sklearn-default.seldon-system:5001\n", + " httpEndpoint: sklearn-default.seldon-system:8000\n", + " svcName: sklearn-default\n", + " sklearn-default-classifier:\n", + " httpEndpoint: sklearn-default-classifier.seldon-system:9000\n", + " svcName: sklearn-default-classifier\n", + " state: Failed\n", + "\n" + ] + } + ], + "source": [ + "!python patch_volumes_1_2.py" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Confirm issues are resolved" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now check first that all of the containers are running" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "seldon-92a927e5e90d7602e08ba9b9304f70e8-6797cc86f7-cv7f9 2/2 Running 0 69s\n", + "sklearn-default-0-classifier-66cf95c445-s6t4x 2/2 Running 0 68s\n", + "NAME READY STATUS RESTARTS AGE\n", + "seldon-controller-manager-7589ff7596-4zqbv 1/1 Running 0 5m2s\n", + "sklearn-default-0-classifier-c86f87c85-xjxf6 2/2 Running 0 68s\n" + ] + } + ], + "source": [ + "!kubectl get pods -n default && kubectl get pods -n seldon-system " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And we confirm that there are no longer any errors in the controller manager logs related to the volumeMount" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/infrastructure/upgrade_1_2_volume_patch/README.md b/examples/infrastructure/upgrade_1_2_volume_patch/README.md new file mode 100644 index 0000000000..12c55a54fe --- /dev/null +++ b/examples/infrastructure/upgrade_1_2_volume_patch/README.md @@ -0,0 +1,558 @@ +## Upgrading from 1.1 to 1.2 Volume Patch + +This notebook contains an overview of how to perform the patch when upgrading from Seldon Core 1.1 into 1.2. + +Note that this is ONLY required if you are performing a rolling upgrade. If you can delete the previous version and install Seldon Core 1.2 you will not need to perform any patching. + +This issue will be fixed in version 1.2.1, so it is recommended to upgrade to this version instead. + +In this notebook we will: +* Install Seldon Core version 1.1 +* Deploy 3 models with varying complexities and specifications +* Perform upgrade +* Observe Issues +* Run patch +* Confirm issues are resolved + +### Install Seldon Core Version 1.1 + + +```bash +%%bash +kubectl create namespace seldon-system || echo "Namespace seldon-system already exists" +helm upgrade --install seldon-core seldon-core-operator \ + --repo https://storage.googleapis.com/seldon-charts \ + --namespace seldon-system \ + --version v1.1.0 \ + --set certManager.enabled="true" \ + --set usageMetrics.enabled=true \ + --set istio.enabled="true" +``` + + Release "seldon-core" does not exist. Installing it now. + NAME: seldon-core + LAST DEPLOYED: Sat Jun 27 10:52:41 2020 + NAMESPACE: seldon-system + STATUS: deployed + REVISION: 1 + TEST SUITE: None + + +Check seldon controller manager is running correctly + + +```python +!kubectl get pods -n seldon-system | grep seldon-controller +``` + + seldon-controller-manager-6978f54b99-xvgvd 1/1 Running 0 7m28s + + +Check no errors in logs + + +```python +!kubectl logs -n seldon-system -l control-plane=seldon-controller-manager | tail -2 +``` + + 2020-06-27T09:59:37.767Z DEBUG controller-runtime.controller Successfully Reconciled {"controller": "seldon-controller-manager", "request": "seldon-system/sklearn"} + 2020-06-27T09:59:37.767Z DEBUG controller-runtime.manager.events Normal {"object": {"kind":"SeldonDeployment","namespace":"seldon-system","name":"sklearn","uid":"4fca069c-eab1-4903-ad23-40517c91207b","apiVersion":"machinelearning.seldon.io/v1","resourceVersion":"1718083"}, "reason": "Updated", "message": "Updated SeldonDeployment \"sklearn\""} + + +### Deploy 3 models + +#### First model is simple sklearn model in default namespace + + +```bash +%%bash +kubectl apply -n default -f - << END +apiVersion: machinelearning.seldon.io/v1alpha2 +kind: SeldonDeployment +metadata: + name: sklearn +spec: + name: iris + predictors: + - graph: + children: [] + implementation: SKLEARN_SERVER + modelUri: gs://seldon-models/sklearn/iris + name: classifier + name: default + replicas: 1 + svcOrchSpec: + env: + - name: SELDON_LOG_LEVEL + value: DEBUG +END +``` + + seldondeployment.machinelearning.seldon.io/sklearn created + + +#### Second model is the same sklaern model but in the seldon-system namespace + + +```bash +%%bash +kubectl apply -n seldon-system -f - << END +apiVersion: machinelearning.seldon.io/v1alpha2 +kind: SeldonDeployment +metadata: + name: sklearn +spec: + name: iris + predictors: + - graph: + children: [] + implementation: SKLEARN_SERVER + modelUri: gs://seldon-models/sklearn/iris + name: classifier + name: default + replicas: 1 + svcOrchSpec: + env: + - name: SELDON_LOG_LEVEL + value: DEBUG +END +``` + + seldondeployment.machinelearning.seldon.io/sklearn created + + +#### Third model is the iris custom model with a mounted volume from a secret + +First we create the secret + + +```bash +%%bash +kubectl create secret generic seldon-test-secret --from-literal=file1.txt=contents --from-literal=file2.txt=morecontents +``` + + secret/seldon-test-secret created + + +Then we deploy the model + + +```bash +%%bash +kubectl apply -f - << END +apiVersion: machinelearning.seldon.io/v1 +kind: SeldonDeployment +metadata: + name: seldon-deployment-example +spec: + name: sklearn-iris-deployment + predictors: + - componentSpecs: + - spec: + volumes: + - name: "secret-mount" + volumeSource: + secret: "seldon-test-secret" + containers: + - image: seldonio/sklearn-iris:0.1 + imagePullPolicy: IfNotPresent + name: sklearn-iris-classifier + volumeMounts: + - name: "secret-mount" + mountPath: "/cert/" + graph: + children: [] + endpoint: + type: REST + name: sklearn-iris-classifier + type: MODEL + name: sklearn-iris-predictor + replicas: 1 +END +``` + + seldondeployment.machinelearning.seldon.io/seldon-deployment-example created + + +Now we wait until they are deployed + + +```python +!kubectl get sdep --all-namespaces +``` + + NAMESPACE NAME AGE + default seldon-deployment-example 39s + default sklearn 60s + seldon-system sklearn 55s + + + +```python +!kubectl get pods -n default && kubectl get pods -n seldon-system +``` + + NAME READY STATUS RESTARTS AGE + seldon-92a927e5e90d7602e08ba9b9304f70e8-8544bc96d-qkm6x 2/2 Running 0 73s + sklearn-default-0-classifier-777f84985b-9tj5r 2/2 Running 0 94s + NAME READY STATUS RESTARTS AGE + seldon-controller-manager-6978f54b99-xvgvd 1/1 Running 0 6m57s + sklearn-default-0-classifier-748c59789b-2lnvh 2/2 Running 0 89s + + +### Perform upgrade to 1.2 + + +```bash +%%bash +helm upgrade --install seldon-core seldon-core-operator \ + --repo https://storage.googleapis.com/seldon-charts \ + --namespace seldon-system \ + --version v1.2.0 \ + --set certManager.enabled="true" \ + --set usageMetrics.enabled=true \ + --set istio.enabled="true" +``` + + Namespace seldon-system already exists + Release "seldon-core" has been upgraded. Happy Helming! + NAME: seldon-core + LAST DEPLOYED: Sat Jun 27 11:03:18 2020 + NAMESPACE: seldon-system + STATUS: deployed + REVISION: 2 + TEST SUITE: None + + + Error from server (AlreadyExists): namespaces "seldon-system" already exists + + +### Observe error + + +```python +!kubectl logs -n seldon-system -l control-plane=seldon-controller-manager | tail -5 +``` + + k8s.io/apimachinery/pkg/util/wait.JitterUntil + /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:153 + k8s.io/apimachinery/pkg/util/wait.Until + /go/pkg/mod/k8s.io/apimachinery@v0.17.2/pkg/util/wait/wait.go:88 + 2020-06-27T10:04:01.898Z DEBUG controller-runtime.manager.events Warning {"object": {"kind":"SeldonDeployment","namespace":"seldon-system","name":"sklearn","uid":"4fca069c-eab1-4903-ad23-40517c91207b","apiVersion":"machinelearning.seldon.io/v1","resourceVersion":"1719032"}, "reason": "InternalError", "message": "Deployment.apps \"sklearn-default-0-classifier\" is invalid: [spec.template.spec.containers[0].volumeMounts[0].name: Not found: \"podinfo\", spec.template.spec.containers[0].volumeMounts[1].mountPath: Invalid value: \"/etc/podinfo\": must be unique]"} + + +### Run Patch + +The error is due a rename on the volumeMounts. We have created the script below which goes through all the seldon deploymetns across all namespaces to rename the volumeMount from podinfo to "seldon-podinfo". + +It is recommended to understand this script fully if this is to be run in prodution as it woudl clash if any existing volume is actually named "podinfo". + + +```python +%%writefile patch_volumes_1_2.py +#!/usr/bin/env python3 + +import yaml +import subprocess +import os +import time + +def run(cmd: str): + cmd_arr = cmd.split() + output = subprocess.Popen(cmd_arr, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT).communicate() + output_str = [out.decode() for out in output if out] + return "\n".join(output_str) + +def patch_volumes_seldon_1_2(): + + namespaces = run("kubectl get ns -o=name") + + for namespace in namespaces.split(): + namespace = namespace.replace("namespace/", "") + sdeps_raw = run(f"kubectl get sdep -o yaml -n {namespace}") + sdeps_dict = yaml.safe_load(sdeps_raw) + sdep_list = sdeps_dict.get("items") + if sdep_list: + for sdep in sdep_list: + name = sdep.get("metadata", {}).get("name") + print(f"Processing {name} in namespace {namespace}") + predictors = sdep.get("spec", {}).get("predictors", []) + for predictor in predictors: + for component_spec in predictor.get("componentSpecs", []): + for container in component_spec.get("spec", {}).get("containers", []): + for volume_mount in container.get("volumeMounts", []): + if volume_mount.get("name") == "podinfo": + print("Patching volume") + volume_mount["name"] = "seldon-podinfo" + + with open("seldon_tmp.yaml", "w") as tmp_file: + yaml.dump(sdep, tmp_file) + run("kubectl apply -f seldon_tmp.yaml") + + print(yaml.dump(sdep)) + os.remove("seldon_tmp.yaml") + + +if __name__ == "__main__": + patch_volumes_seldon_1_2() + +``` + + Overwriting patch_volumes_1_2.py + + +Run script + + +```python +!python patch_volumes_1_2.py +``` + + Processing seldon-deployment-example in namespace default + Patching volume + apiVersion: machinelearning.seldon.io/v1 + kind: SeldonDeployment + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: '{"apiVersion":"machinelearning.seldon.io/v1","kind":"SeldonDeployment","metadata":{"annotations":{},"name":"seldon-deployment-example","namespace":"default"},"spec":{"name":"sklearn-iris-deployment","predictors":[{"componentSpecs":[{"spec":{"containers":[{"image":"seldonio/sklearn-iris:0.1","imagePullPolicy":"IfNotPresent","name":"sklearn-iris-classifier","volumeMounts":[{"mountPath":"/cert/","name":"secret-mount"}]}],"volumes":[{"name":"secret-mount","volumeSource":{"secret":"seldon-test-secret"}}]}}],"graph":{"children":[],"endpoint":{"type":"REST"},"name":"sklearn-iris-classifier","type":"MODEL"},"name":"sklearn-iris-predictor","replicas":1}]}} + + ' + creationTimestamp: '2020-06-27T09:58:26Z' + generation: 1 + name: seldon-deployment-example + namespace: default + resourceVersion: '1719036' + selfLink: /apis/machinelearning.seldon.io/v1/namespaces/default/seldondeployments/seldon-deployment-example + uid: 8a15eb91-e614-41d9-9d0e-abc191d3a417 + spec: + name: sklearn-iris-deployment + predictors: + - componentSpecs: + - metadata: + creationTimestamp: null + spec: + containers: + - image: seldonio/sklearn-iris:0.1 + imagePullPolicy: IfNotPresent + name: sklearn-iris-classifier + ports: + - containerPort: 6000 + name: metrics + protocol: TCP + resources: {} + volumeMounts: + - mountPath: /cert/ + name: secret-mount + - mountPath: /etc/podinfo + name: seldon-podinfo + volumes: + - name: secret-mount + engineResources: {} + graph: + endpoint: + service_host: localhost + service_port: 9000 + type: REST + implementation: UNKNOWN_IMPLEMENTATION + name: sklearn-iris-classifier + type: MODEL + labels: + version: sklearn-iris-predictor + name: sklearn-iris-predictor + replicas: 1 + svcOrchSpec: {} + status: + address: + url: http://seldon-deployment-example-sklearn-iris-predictor.default.svc.cluster.local:8000/api/v1.0/predictions + deploymentStatus: + seldon-92a927e5e90d7602e08ba9b9304f70e8: + availableReplicas: 1 + replicas: 1 + description: 'Deployment.apps "seldon-92a927e5e90d7602e08ba9b9304f70e8" is invalid: + [spec.template.spec.containers[0].volumeMounts[1].name: Not found: "podinfo", + spec.template.spec.containers[0].volumeMounts[2].mountPath: Invalid value: "/etc/podinfo": + must be unique]' + replicas: 1 + serviceStatus: + seldon-d0934233541ef6b732c88680f8a0e94f: + httpEndpoint: seldon-d0934233541ef6b732c88680f8a0e94f.default:9000 + svcName: seldon-d0934233541ef6b732c88680f8a0e94f + seldon-deployment-example-sklearn-iris-predictor: + grpcEndpoint: seldon-deployment-example-sklearn-iris-predictor.default:5001 + httpEndpoint: seldon-deployment-example-sklearn-iris-predictor.default:8000 + svcName: seldon-deployment-example-sklearn-iris-predictor + state: Failed + + Processing sklearn in namespace default + Patching volume + apiVersion: machinelearning.seldon.io/v1 + kind: SeldonDeployment + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: '{"apiVersion":"machinelearning.seldon.io/v1alpha2","kind":"SeldonDeployment","metadata":{"annotations":{},"name":"sklearn","namespace":"default"},"spec":{"name":"iris","predictors":[{"graph":{"children":[],"implementation":"SKLEARN_SERVER","modelUri":"gs://seldon-models/sklearn/iris","name":"classifier"},"name":"default","replicas":1,"svcOrchSpec":{"env":[{"name":"SELDON_LOG_LEVEL","value":"DEBUG"}]}}]}} + + ' + creationTimestamp: '2020-06-27T09:58:05Z' + generation: 1 + name: sklearn + namespace: default + resourceVersion: '1719025' + selfLink: /apis/machinelearning.seldon.io/v1/namespaces/default/seldondeployments/sklearn + uid: 4f44a5dc-8da4-45ba-8ace-00e51643c7ff + spec: + name: iris + predictors: + - componentSpecs: + - metadata: + creationTimestamp: '2020-06-27T09:58:05Z' + spec: + containers: + - image: seldonio/sklearnserver_rest:0.3 + name: classifier + ports: + - containerPort: 6000 + name: metrics + protocol: TCP + resources: {} + volumeMounts: + - mountPath: /etc/podinfo + name: seldon-podinfo + engineResources: {} + graph: + endpoint: + service_host: localhost + service_port: 9000 + type: REST + implementation: SKLEARN_SERVER + modelUri: gs://seldon-models/sklearn/iris + name: classifier + type: MODEL + labels: + version: default + name: default + replicas: 1 + svcOrchSpec: + env: + - name: SELDON_LOG_LEVEL + value: DEBUG + status: + address: + url: http://sklearn-default.default.svc.cluster.local:8000/api/v1.0/predictions + deploymentStatus: + sklearn-default-0-classifier: + availableReplicas: 1 + replicas: 1 + description: 'Deployment.apps "sklearn-default-0-classifier" is invalid: [spec.template.spec.containers[0].volumeMounts[0].name: + Not found: "podinfo", spec.template.spec.containers[0].volumeMounts[1].mountPath: + Invalid value: "/etc/podinfo": must be unique]' + replicas: 1 + serviceStatus: + sklearn-default: + grpcEndpoint: sklearn-default.default:5001 + httpEndpoint: sklearn-default.default:8000 + svcName: sklearn-default + sklearn-default-classifier: + httpEndpoint: sklearn-default-classifier.default:9000 + svcName: sklearn-default-classifier + state: Failed + + Processing sklearn in namespace seldon-system + Patching volume + apiVersion: machinelearning.seldon.io/v1 + kind: SeldonDeployment + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: '{"apiVersion":"machinelearning.seldon.io/v1alpha2","kind":"SeldonDeployment","metadata":{"annotations":{},"name":"sklearn","namespace":"seldon-system"},"spec":{"name":"iris","predictors":[{"graph":{"children":[],"implementation":"SKLEARN_SERVER","modelUri":"gs://seldon-models/sklearn/iris","name":"classifier"},"name":"default","replicas":1,"svcOrchSpec":{"env":[{"name":"SELDON_LOG_LEVEL","value":"DEBUG"}]}}]}} + + ' + creationTimestamp: '2020-06-27T09:58:10Z' + generation: 1 + name: sklearn + namespace: seldon-system + resourceVersion: '1719032' + selfLink: /apis/machinelearning.seldon.io/v1/namespaces/seldon-system/seldondeployments/sklearn + uid: 4fca069c-eab1-4903-ad23-40517c91207b + spec: + name: iris + predictors: + - componentSpecs: + - metadata: + creationTimestamp: '2020-06-27T09:58:10Z' + spec: + containers: + - image: seldonio/sklearnserver_rest:0.3 + name: classifier + ports: + - containerPort: 6000 + name: metrics + protocol: TCP + resources: {} + volumeMounts: + - mountPath: /etc/podinfo + name: seldon-podinfo + engineResources: {} + graph: + endpoint: + service_host: localhost + service_port: 9000 + type: REST + implementation: SKLEARN_SERVER + modelUri: gs://seldon-models/sklearn/iris + name: classifier + type: MODEL + labels: + version: default + name: default + replicas: 1 + svcOrchSpec: + env: + - name: SELDON_LOG_LEVEL + value: DEBUG + status: + address: + url: http://sklearn-default.seldon-system.svc.cluster.local:8000/api/v1.0/predictions + deploymentStatus: + sklearn-default-0-classifier: + availableReplicas: 1 + replicas: 1 + description: 'Deployment.apps "sklearn-default-0-classifier" is invalid: [spec.template.spec.containers[0].volumeMounts[0].name: + Not found: "podinfo", spec.template.spec.containers[0].volumeMounts[1].mountPath: + Invalid value: "/etc/podinfo": must be unique]' + replicas: 1 + serviceStatus: + sklearn-default: + grpcEndpoint: sklearn-default.seldon-system:5001 + httpEndpoint: sklearn-default.seldon-system:8000 + svcName: sklearn-default + sklearn-default-classifier: + httpEndpoint: sklearn-default-classifier.seldon-system:9000 + svcName: sklearn-default-classifier + state: Failed + + + +### Confirm issues are resolved + +We can now check first that all of the containers are running + + +```python +!kubectl get pods -n default && kubectl get pods -n seldon-system +``` + + NAME READY STATUS RESTARTS AGE + seldon-92a927e5e90d7602e08ba9b9304f70e8-6797cc86f7-cv7f9 2/2 Running 0 69s + sklearn-default-0-classifier-66cf95c445-s6t4x 2/2 Running 0 68s + NAME READY STATUS RESTARTS AGE + seldon-controller-manager-7589ff7596-4zqbv 1/1 Running 0 5m2s + sklearn-default-0-classifier-c86f87c85-xjxf6 2/2 Running 0 68s + + +And we confirm that there are no longer any errors in the controller manager logs related to the volumeMount + + +```python + +``` diff --git a/examples/infrastructure/upgrade_1_2_volume_patch/patch_volumes_1_2.py b/examples/infrastructure/upgrade_1_2_volume_patch/patch_volumes_1_2.py new file mode 100644 index 0000000000..e02fd58437 --- /dev/null +++ b/examples/infrastructure/upgrade_1_2_volume_patch/patch_volumes_1_2.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import yaml +import subprocess +import os +import time + + +def run(cmd: str): + cmd_arr = cmd.split() + output = subprocess.Popen( + cmd_arr, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ).communicate() + output_str = [out.decode() for out in output if out] + return "\n".join(output_str) + + +def patch_volumes_seldon_1_2(): + + namespaces = run("kubectl get ns -o=name") + + for namespace in namespaces.split(): + namespace = namespace.replace("namespace/", "") + sdeps_raw = run(f"kubectl get sdep -o yaml -n {namespace}") + sdeps_dict = yaml.safe_load(sdeps_raw) + sdep_list = sdeps_dict.get("items") + if sdep_list: + for sdep in sdep_list: + name = sdep.get("metadata", {}).get("name") + print(f"Processing {name} in namespace {namespace}") + predictors = sdep.get("spec", {}).get("predictors", []) + for predictor in predictors: + for component_spec in predictor.get("componentSpecs", []): + for container in component_spec.get("spec", {}).get( + "containers", [] + ): + for volume_mount in container.get("volumeMounts", []): + if volume_mount.get("name") == "podinfo": + print("Patching volume") + volume_mount["name"] = "seldon-podinfo" + + with open("seldon_tmp.yaml", "w") as tmp_file: + yaml.dump(sdep, tmp_file) + run("kubectl apply -f seldon_tmp.yaml") + + print(yaml.dump(sdep)) + os.remove("seldon_tmp.yaml") + + +if __name__ == "__main__": + patch_volumes_seldon_1_2()