diff --git a/ci_build_and_push_images.sh b/ci_build_and_push_images.sh index 70d2e5d16b..e5a85bc92d 100755 --- a/ci_build_and_push_images.sh +++ b/ci_build_and_push_images.sh @@ -68,10 +68,8 @@ function build_push_engine { function build_push_mock { make \ -C examples/models/mean_classifier \ - build_rest \ - build_grpc \ - push_rest \ - push_grpc + build \ + push MOCK_MODEL_EXIT_VALUE=$? } @@ -94,32 +92,32 @@ function build_push_request_logger { function build_push_sklearnserver { make \ -C servers/sklearnserver \ - build_all \ - push_all + build \ + push SKLEARN_EXIT_VALUE=$? } function build_push_mlflowserver { make \ -C servers/mlflowserver \ - build_all \ - push_all + build \ + push MLFLOW_EXIT_VALUE=$? } function build_push_xgboostserver { make \ -C servers/xgboostserver \ - build_all \ - push_all + build \ + push XGBOOST_EXIT_VALUE=$? } function build_push_tfproxy { make \ - -C integrations/tfserving \ - build_all \ - push_all + -C integrations/tfserving_proxy \ + build \ + push TFPROXY_EXIT_VALUE=$? } @@ -139,6 +137,14 @@ function build_push_storage_initializer { STORAGE_INITIALIZER_EXIT_VALUE=$? } +function build_push_mab { + make \ + -C components/routers/epsilon-greedy \ + build \ + push + MAB_EXIT_VALUE=$? +} + build_push_python build_push_operator @@ -153,6 +159,7 @@ build_push_xgboostserver build_push_tfproxy build_push_alibi_explainer build_push_storage_initializer +build_push_mab ####################################### # EXIT STOPS COMMANDS FROM HERE ONWARDS @@ -171,6 +178,7 @@ echo "Mock model exit value: $MOCK_MODEL_EXIT_VALUE" echo "Alibi Detect exit value: $ALIBI_DETECT_EXIT_VALUE" echo "Request Logger exit value: $LOGGER_EXIT_VALUE" echo "Tensorflow Proxy exit value: $TFPROXY_EXIT_VALUE" +echo "MAB exit value: $MAB_EXIT_VALUE" exit $((${PYTHON_EXIT_VALUE} \ + ${OPERATOR_EXIT_VALUE} \ @@ -184,6 +192,7 @@ exit $((${PYTHON_EXIT_VALUE} \ + ${XGBOOST_EXIT_VALUE} \ + ${TFPROXY_EXIT_VALUE} \ + ${STORAGE_INITIALIZER_EXIT_VALUE} \ + + ${MAB_EXIT_VALUE} \ + ${EXPLAIN_EXIT_VALUE})) diff --git a/components/alibi-explain-server/tests/test_anchor_tabular.py b/components/alibi-explain-server/tests/test_anchor_tabular.py index 573448dcc3..38f533e128 100644 --- a/components/alibi-explain-server/tests/test_anchor_tabular.py +++ b/components/alibi-explain-server/tests/test_anchor_tabular.py @@ -26,7 +26,7 @@ import json from .utils import SKLearnServer ADULT_EXPLAINER_URI = "gs://seldon-models/sklearn/income/alibi/0.4.0" -ADULT_MODEL_URI = "gs://seldon-models/sklearn/income/model-0.23.2" +ADULT_MODEL_URI = "gs://seldon-models/sklearn/income/model" EXPLAINER_FILENAME = "explainer.dill" diff --git a/components/routers/epsilon-greedy/Makefile b/components/routers/epsilon-greedy/Makefile index 3d2253db6e..4413136a30 100644 --- a/components/routers/epsilon-greedy/Makefile +++ b/components/routers/epsilon-greedy/Makefile @@ -1,14 +1,12 @@ -IMAGE_VERSION=1.3 +VERSION := $(shell cat ../../../version.txt) IMAGE_NAME=seldonio/mab_epsilon_greedy +KIND_NAME ?= kind -build_rest: - s2i build . -E environment_rest seldonio/seldon-core-s2i-python3:0.15 $(IMAGE_NAME)_rest:$(IMAGE_VERSION) +build: + s2i build . -E environment seldonio/seldon-core-s2i-python3:${VERSION} $(IMAGE_NAME):$(VERSION) -push_to_dockerhub_rest: - docker push $(IMAGE_NAME)_rest:$(IMAGE_VERSION) +push: + docker push $(IMAGE_NAME):$(VERSION) -build_grpc: - s2i build . -E environment_grpc seldonio/seldon-core-s2i-python3:0.15 $(IMAGE_NAME)_grpc:$(IMAGE_VERSION) - -push_to_dockerhub_grpc: - docker push $(IMAGE_NAME)_grpc:$(IMAGE_VERSION) +kind_load: build + kind load -v 3 docker-image ${IMAGE_NAME}:${VERSION} --name ${KIND_NAME} diff --git a/components/routers/epsilon-greedy/environment_grpc b/components/routers/epsilon-greedy/environment similarity index 80% rename from components/routers/epsilon-greedy/environment_grpc rename to components/routers/epsilon-greedy/environment index 1a7970f916..a4e192122e 100644 --- a/components/routers/epsilon-greedy/environment_grpc +++ b/components/routers/epsilon-greedy/environment @@ -1,4 +1,3 @@ MODEL_NAME=EpsilonGreedy -API_TYPE=GRPC SERVICE_TYPE=ROUTER PERSISTENCE=0 diff --git a/components/routers/epsilon-greedy/environment_rest b/components/routers/epsilon-greedy/environment_rest deleted file mode 100644 index 9f3afdd4c6..0000000000 --- a/components/routers/epsilon-greedy/environment_rest +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=EpsilonGreedy -API_TYPE=REST -SERVICE_TYPE=ROUTER -PERSISTENCE=0 diff --git a/doc/source/examples/backwards_compatibility.nblink b/doc/source/examples/backwards_compatibility.nblink new file mode 100644 index 0000000000..b4034b9638 --- /dev/null +++ b/doc/source/examples/backwards_compatibility.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../../notebooks/backwards_compatability.ipynb" +} diff --git a/doc/source/examples/istio.md b/doc/source/examples/istio.md index ca2fb95a58..4df09681c7 100644 --- a/doc/source/examples/istio.md +++ b/doc/source/examples/istio.md @@ -12,6 +12,3 @@ The addition of Istio is complementary to Seldon and is illustrated below where ![svc-graph-istio](./svc-graph-istio.png) -## Worked Examples - -[An example step-by-step guide to canary deployments using Istio and Seldon is provided](./istio_canary.html) diff --git a/doc/source/examples/istio_canary.nblink b/doc/source/examples/istio_canary.nblink deleted file mode 100644 index 43dd5a5013..0000000000 --- a/doc/source/examples/istio_canary.nblink +++ /dev/null @@ -1,3 +0,0 @@ -{ - "path": "../../../examples/istio/canary_update/canary.ipynb" -} diff --git a/doc/source/examples/notebooks.rst b/doc/source/examples/notebooks.rst index c08cb24d99..3fe249b71c 100644 --- a/doc/source/examples/notebooks.rst +++ b/doc/source/examples/notebooks.rst @@ -140,7 +140,6 @@ Ingress Ambassador Shadow Ambassador Headers Ambassador Custom Config - Istio Canary Istio Examples Infrastructure @@ -161,4 +160,4 @@ Benchmarking and Load Tests Service Orchestrator Tensorflow Argo Workflows Benchmarking - + Backwards Compatability Tests diff --git a/doc/source/java-jni/README.md b/doc/source/java-jni/README.md index c94d3352ee..9e8e5ce24b 100644 --- a/doc/source/java-jni/README.md +++ b/doc/source/java-jni/README.md @@ -132,7 +132,6 @@ Define the core parameters needed by our Java S2I images to wrap your model. An example is: ```bash -API_TYPE=REST SERVICE_TYPE=MODEL JAVA_IMPORT_PATH=io.seldon.example.model.ExampleModelHandler ``` @@ -213,10 +212,6 @@ Import path for your Java model implementation. For instance, in the example above, this would be `io.seldon.example.model.ExampleModelHandler`. -### API_TYPE - -API type to create. -Can be REST or GRPC. ### SERVICE_TYPE diff --git a/doc/source/python/index.rst b/doc/source/python/index.rst index dcc89ee91d..27f4b13b7f 100644 --- a/doc/source/python/index.rst +++ b/doc/source/python/index.rst @@ -12,12 +12,11 @@ You can use the following links to navigate the Python seldon-core module: .. toctree:: :maxdepth: 1 - Seldon Core Python Module - Your python class - Wrap using S2I - Wrap using Docker - Seldon Python Client - Seldon Python Server - Python API reference - + Install the Seldon Core Python module + Creating your Python inference class + Create image with S2I + Create image with a Dockerfile + Seldon Python server configuration + Calling the Seldon API with the Seldon Python client + Python API reference diff --git a/doc/source/python/python_component.md b/doc/source/python/python_component.md index 1e48b43200..c10d6fec4a 100644 --- a/doc/source/python/python_component.md +++ b/doc/source/python/python_component.md @@ -264,7 +264,7 @@ spec: ## Low level Methods -If you want more control you can provide a low-level methods that will provide as input the raw proto buffer payloads. The signatures for these are shown below for release `sedon_core>=0.2.6.1`: +If you want more control you can provide a low-level methods that will provide as input the raw proto buffer payloads. The signatures for these are shown below for release `seldon_core>=0.2.6.1`: ```python def predict_raw(self, msg: prediction_pb2.SeldonMessage) -> prediction_pb2.SeldonMessage: diff --git a/doc/source/python/python_module.md b/doc/source/python/python_module.md index 96c6d578b3..15dcc74551 100644 --- a/doc/source/python/python_module.md +++ b/doc/source/python/python_module.md @@ -71,46 +71,6 @@ Keep in mind that this will include some dependencies which may not be used. Therefore, unless necessary, we recommend most users to install the default distribution of `seldon-core` as [documented above](#install). -## Seldon Core Microservices - -Seldon allows you to easily take your runtime inference code and create a Docker container that can be managed by Seldon Core. Follow the [S2I instructions](../wrappers/python.md) to wrap your code. - -You can also create your own image and utilise the `seldon-core-microservice` executable to run your model code. - - -## Seldon Core Python API Client - -The python package contains a module that provides a reference python client for the internal Seldon Core microservice API and the external APIs. More specifically it provides: - - * Internal microservice API - * Make REST or gRPC calls - * Test all methods: `predict`, `transform-input`, `transform-output`, `route`, `aggregate` - * Provide a numpy array, binary data or string data as payload or get random data generated as payload for given shape - * Send data as tensor, TFTensor or ndarray - * External API - * Make REST or gRPC calls - * Call the API via Ambassador, Istio or Seldon's OAUTH API gateway. - * Test `predict` or `feedback` endpoints - * Provide a numpy array, binary data or string data as payload or get random data generated as payload for given shape - * Send data as tensor, TFTensor or ndarray - -Basic usage of the client is to create a `SeldonClient` object first. For example for a Seldon Deployment called "mymodel" running in the namespace `seldon` with Ambassador endpoint at "localhost:8003" (i.e., via port-forwarding): - -```python -from seldon_core.seldon_client import SeldonClient -sc = SeldonClient(deployment_name="mymodel",namespace="seldon", gateway_endpoint="localhost:8003") -``` - -Then make calls of various types. For example, to make a random prediction via the Ambassador gateway using REST: - -```python -r = sc.predict(gateway="ambassador",transport="rest") -print(r) -``` - -Examples of using the `seldon_client` module can be found in the [example notebook](../examples/helm_examples.html). - -The API docs can be found [here](./api/seldon_core.html#module-seldon_core.seldon_client). ## Troubleshooting @@ -146,3 +106,9 @@ also works: ```bash $ pip install azure-storage-blob==2.1.0 seldon-core ``` + +## Next Steps + +[Create your python inference class](python_component.md) + + diff --git a/doc/source/python/python_server.md b/doc/source/python/python_server.md index 6fd5a3224f..ad1ab9f7cc 100644 --- a/doc/source/python/python_server.md +++ b/doc/source/python/python_server.md @@ -1,4 +1,10 @@ -# Seldon Python Server +# Seldon Python Server Configuration + + * [Workers](#workers) + * [Threads](#threads) + * [Flask Development Server](#development-server) + * [Server Configuration](#configuration) + To serve your component, Seldon's Python wrapper will use [Gunicorn](https://gunicorn.org/) under the hood by default. diff --git a/doc/source/python/python_wrapping_docker.md b/doc/source/python/python_wrapping_docker.md index d79d0d17c1..377dfc11d9 100644 --- a/doc/source/python/python_wrapping_docker.md +++ b/doc/source/python/python_wrapping_docker.md @@ -65,11 +65,10 @@ EXPOSE 5000 # Define environment variable ENV MODEL_NAME MyModel -ENV API_TYPE REST ENV SERVICE_TYPE MODEL ENV PERSISTENCE 0 -CMD exec seldon-core-microservice $MODEL_NAME $API_TYPE --service-type $SERVICE_TYPE --persistence $PERSISTENCE +CMD exec seldon-core-microservice $MODEL_NAME --service-type $SERVICE_TYPE --persistence $PERSISTENCE ``` @@ -87,10 +86,6 @@ The required environment variables understood by the builder image are explained ### MODEL_NAME The name of the class containing the model. Also the name of the python file which will be imported. -### API_TYPE - -API type to create. Can be REST or GRPC - ### SERVICE_TYPE The service type being created. Available options are: @@ -157,7 +152,6 @@ These arguments can be set when deploying in a Seldon Deployment. An example can { "graph": { "name": "tfserving-proxy", - "endpoint": { "type": "REST" }, "type": "MODEL", "children": [], "parameters": [ diff --git a/doc/source/python/python_wrapping_s2i.md b/doc/source/python/python_wrapping_s2i.md index 23c8a229b2..603991a4d8 100644 --- a/doc/source/python/python_wrapping_s2i.md +++ b/doc/source/python/python_wrapping_s2i.md @@ -35,39 +35,8 @@ To use our s2i builder image to package your python model you will need: We will go into detail for each of these steps: ### Python file -Your source code should contain a python file which defines a class of the same name as the file. For example, looking at our skeleton python model file at `wrappers/s2i/python/test/model-template-app/MyModel.py`: -```python -class MyModel(object): - """ - Model template. - You can load your model parameters in __init__ from a location accessible at runtime. - """ - - def __init__(self): - """ - Add any initialization parameters. - These will be passed at runtime from the graph definition parameters defined in your seldondeployment kubernetes resource manifest. - """ - print("Initializing") - - def predict(self,X,features_names): - """ - Return a prediction. - - Parameters - ---------- - X : array-like - feature_names : array of feature names (optional) - """ - print("Predict called - will run identity function") - return X -``` - - * The file is called MyModel.py and it defines a class MyModel - * The class contains a predict method that takes an array (numpy) X and feature_names and returns an array of predictions. - * You can add any required initialization inside the class init method. - * Your return array should be at least 2-dimensional. +Your source code should contain a python file which defines a class of the same name as the file. For further details see [details on creating your python class](python_component.md) ### Dependencies @@ -119,13 +88,14 @@ Define the core parameters needed by our python builder image to wrap your model ```bash MODEL_NAME=MyModel -API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0 ``` These values can also be provided or overridden on the command line when building the image. +See below for the possible keys and values for this file. + ## Step 3 - Build your image Use `s2i build` to create your Docker image from source code. You will need Docker installed on the machine and optionally git if your source code is in a public git repo. You can choose from three python builder images @@ -181,10 +151,6 @@ The required environment variables understood by the builder image are explained The name of the class containing the model. Also the name of the python file which will be imported. -### API_TYPE - -API type to create. Can be REST or GRPC - ### SERVICE_TYPE The service type being created. Available options are: @@ -304,7 +270,7 @@ The allowable `type` values for the parameters are defined in the [proto buffer ### Local Python Dependencies -`from version 0.5-SNAPSHOT` +`from version 0.5` To use a private repository for installing Python dependencies use the following build command: diff --git a/doc/source/reference/upgrading.md b/doc/source/reference/upgrading.md index e82e4e4ea6..4bfbd67289 100644 --- a/doc/source/reference/upgrading.md +++ b/doc/source/reference/upgrading.md @@ -6,6 +6,14 @@ If you were running our Openshift 0.4.2 certified operator and are looking to up Make sure you also [read the CHANGELOG](./changelog.html) to see the detailed features and bug-fixes in each version. +## Upgrading to 1.5 + +### REST and gRPC + +To take advantage of the ability to handle both REST and gRPC on any deployed model python model images will need to be recreated using the 1.5 python wrapper. If they are not updated they will only expose the protocol they were orginally wrapped for. + +You can use and extend the [backwards compatibility notebook](../examples/backwards_compatibility.html) to check your deployments will work if you do not intend to upgrade them. + ## Upgrading to 1.3 ### Breaking Changes diff --git a/doc/source/tutorials/openshift_s2i.md b/doc/source/tutorials/openshift_s2i.md index ed26b0f2db..4feb54721f 100644 --- a/doc/source/tutorials/openshift_s2i.md +++ b/doc/source/tutorials/openshift_s2i.md @@ -85,7 +85,6 @@ To allow the s2i builder image to correctly package the component the data scien ```bash MODEL_NAME=MyModel -API_TYPE=REST SERVICE_TYPE=MODEL ``` @@ -96,82 +95,6 @@ s2i build seldonio/seldon-core-s2i-python3 s2i build seldonio/seldon-core-s2i-python3 ``` -## R - -R is a popular statistical language which provides many machine learning related packages. - -To use the seldon s2i builder image to package an R model the requirements are: - -- An R file which provides an S3 class for your model via an `initialise_seldon` function and that has appropriate generics for the component, e.g. predict for a model. -- An optional install.R to be run to install any libraries needed -- .s2i/environment - model definitions used by the s2i builder to correctly wrap your model - -The data scientist's source code should contain an R file which defines an S3 class for their model. For example, - -```R -library(methods) - -predict.mymodel <- function(mymodel,newdata=list()) { - write("MyModel predict called", stdout()) - newdata -} - -new_mymodel <- function() { - structure(list(), class = "mymodel") -} - -initialise_seldon <- function(params) { - new_mymodel() -} -``` - -The above contains: - -- A `seldon_initialise` function that creates an S3 class for the model via a constructor `new_mymodel`. This will be called on startup and you can run any configuration the model needs. -- A generic `predict` function is created for my model class. This will be called with a `newdata` field with the `data.frame` to be predicted. - -An `install.R` with any software dependencies required. For example: - -```R -install.packages('rpart') -``` - -Finally, as with all cases the builder image needs a few environment variables to be set to correctly package the R model. An example is: - -```bash -MODEL_NAME=MyModel -API_TYPE=REST -SERVICE_TYPE=MODEL -``` - -These values can also be provided in an .s2i/environment file with the source code or overridden on the command line when building the image. - -Once these steps are done we can use `s2i build` to create the Docker-formatted image from the source code. - -```bash -s2i build seldonio/seldon-core-s2i-r -s2i build seldonio/seldon-core-s2i-r -``` - -An example invocation using the test template model inside seldon-core: - -```bash -s2i build https://github.com/seldonio/seldon-core.git --context-dir=incubating/wrappers/s2i/R/test/model-template-app seldonio/seldon-core-s2i-r seldon-core-template-model -``` - -### Java - -There are several popular machine learning libraries in Java including Spark, H2O and DL4J. Seldon-core also provides builder images for Java. To accomplish this we provide a Java library seldon-core-wrappers that can be included in a Maven Spring project to allow a Java component to be easily wrapped. - -To use the Seldon-Core s2i builder image to package a Java model the data scientist will need: - -- A Maven project that depends on the `io.seldon.wrapper` library -- A Spring Boot configuration class -- A class that implements `io.seldon.wrapper.SeldonPredictionService` for the type of component you are creating -- An optional .s2i/environment - model definitions used by the s2i builder to correctly wrap your model - -More details can be found in the seldon-core docs. - ## Summary By utilizing Openshift's source-to-image tool data scientists can easily build Docker-formatted images for their runtime components to be deployed at scale using seldon-core. This allows data science teams to use the best machine learning tool for the task and deploy the resulting model in a consistent manner. The seldon-core project is working on providing full Openshift integration in the near future so that Enterprise customers can easily utilize machine learning models within their organisation. diff --git a/examples/ambassador/canary/.gitignore b/examples/ambassador/canary/.gitignore new file mode 100644 index 0000000000..12a9651e81 --- /dev/null +++ b/examples/ambassador/canary/.gitignore @@ -0,0 +1,2 @@ +canary.yaml +model.yaml \ No newline at end of file diff --git a/examples/ambassador/canary/ambassador_canary.ipynb b/examples/ambassador/canary/ambassador_canary.ipynb index c98490a9ca..17325dd1fd 100644 --- a/examples/ambassador/canary/ambassador_canary.ipynb +++ b/examples/ambassador/canary/ambassador_canary.ipynb @@ -18,22 +18,74 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "VERSION=!cat ../../../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -45,27 +97,68 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ - "!pygmentize model.json" + "%%writetemplate model.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example\n", + "spec:\n", + " name: canary-example\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: main\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example created\r\n" + ] + } + ], "source": [ - "!kubectl create -f model.json" + "!kubectl create -f model.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-main-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-main-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[0].metadata.name}')" ] @@ -79,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -96,9 +189,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.5531305252881753\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.08599584459790496]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\")\n", "assert(r.success==True)\n", @@ -116,27 +230,85 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ - "!pygmentize canary.json" + "%%writetemplate canary.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example\n", + "spec:\n", + " name: canary-example\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: main\n", + " replicas: 1\n", + " traffic: 75\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: canary\n", + " replicas: 1\n", + " traffic: 25\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f canary.json" + "!kubectl apply -f canary.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"example-canary-0-classifier\" successfully rolled out\n", + "deployment \"example-main-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[0].metadata.name}')\n", "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[1].metadata.name}')" @@ -151,16 +323,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.08077854180978328\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.0554153788429373]}}, 'meta': {}}" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "sc.predict(gateway=\"ambassador\",transport=\"rest\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +369,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -182,7 +378,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -191,9 +387,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.2948717948717949\n" + ] + } + ], "source": [ "canary_percentage=float(canary_count[0])/float(default_count[0])\n", "print(canary_percentage)\n", @@ -202,11 +406,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f canary.json" + "!kubectl delete -f canary.yaml" ] }, { diff --git a/examples/ambassador/canary/canary.json b/examples/ambassador/canary/canary.json deleted file mode 100644 index 3bcc6054ad..0000000000 --- a/examples/ambassador/canary/canary.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "example" - }, - "spec": { - "name": "canary-example", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "main", - "replicas": 1, - "traffic": 75 - }, - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "canary", - "replicas": 1, - "traffic": 25 - } - ] - } -} - diff --git a/examples/ambassador/canary/model.json b/examples/ambassador/canary/model.json deleted file mode 100644 index 369a44622f..0000000000 --- a/examples/ambassador/canary/model.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "example" - }, - "spec": { - "name": "canary-example", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "main", - "replicas": 1 - } - ] - } -} - diff --git a/examples/ambassador/custom/.gitignore b/examples/ambassador/custom/.gitignore new file mode 100644 index 0000000000..bf34305ba0 --- /dev/null +++ b/examples/ambassador/custom/.gitignore @@ -0,0 +1 @@ +model_custom_ambassador.yaml diff --git a/examples/ambassador/custom/ambassador_custom.ipynb b/examples/ambassador/custom/ambassador_custom.ipynb index d16ca890a6..f43e7f1e72 100644 --- a/examples/ambassador/custom/ambassador_custom.ipynb +++ b/examples/ambassador/custom/ambassador_custom.ipynb @@ -9,6 +9,42 @@ "This notebook shows how you can deploy Seldon Deployments with custom Ambassador configuration." ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "VERSION=!cat ../../../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -48,27 +84,80 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "!pygmentize model_custom_ambassador.json" + "%%writetemplate model_custom_ambassador.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example-custom\n", + "spec:\n", + " annotations:\n", + " seldon.io/ambassador-config: 'apiVersion: ambassador/v1\n", + "\n", + " kind: Mapping\n", + "\n", + " name: seldon_example_rest_mapping\n", + "\n", + " prefix: /mycompany/ml/\n", + "\n", + " service: example-custom-single.seldon:8000\n", + "\n", + " timeout_ms: 3000'\n", + " name: production-model\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: single\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-custom created\r\n" + ] + } + ], "source": [ - "!kubectl create -f model_custom_ambassador.json" + "!kubectl create -f model_custom_ambassador.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-custom-single-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-custom-single-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-custom -o jsonpath='{.items[0].metadata.name}')" ] @@ -82,7 +171,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -99,9 +188,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.7361019060122931\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.10150940716895476]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\",gateway_prefix=\"/mycompany/ml\")\n", "assert(r.success==True)\n", @@ -110,9 +220,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "error: the path \"model_custom_ambassador.json\" does not exist\r\n" + ] + } + ], "source": [ "!kubectl delete -f model_custom_ambassador.json" ] diff --git a/examples/ambassador/custom/model_custom_ambassador.json b/examples/ambassador/custom/model_custom_ambassador.json deleted file mode 100644 index 32b9fc076f..0000000000 --- a/examples/ambassador/custom/model_custom_ambassador.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "example-custom" - }, - "spec": { - "name": "production-model", - "annotations": { - "seldon.io/ambassador-config":"apiVersion: ambassador/v1\nkind: Mapping\nname: seldon_example_rest_mapping\nprefix: /mycompany/ml/\nservice: example-custom-single.seldon:8000\ntimeout_ms: 3000" - }, - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "single", - "replicas": 1 - } - ] - } -} - diff --git a/examples/ambassador/headers/.gitignore b/examples/ambassador/headers/.gitignore new file mode 100644 index 0000000000..e4ef638ca4 --- /dev/null +++ b/examples/ambassador/headers/.gitignore @@ -0,0 +1,2 @@ +model_with_header.yaml +model.yaml \ No newline at end of file diff --git a/examples/ambassador/headers/ambassador_headers.ipynb b/examples/ambassador/headers/ambassador_headers.ipynb index 7c92f85d06..3c5707b353 100644 --- a/examples/ambassador/headers/ambassador_headers.ipynb +++ b/examples/ambassador/headers/ambassador_headers.ipynb @@ -20,22 +20,74 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "VERSION=!cat ../../../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -47,27 +99,68 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "!pygmentize model.json" + "%%writetemplate model.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example\n", + "spec:\n", + " name: production-model\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: single\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example created\r\n" + ] + } + ], "source": [ - "!kubectl create -f model.json" + "!kubectl create -f model.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-single-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-single-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[0].metadata.name}')" ] @@ -81,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -98,9 +191,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.6683377893898669\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.0954937100180911]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\")\n", "assert(r.success==True)\n", @@ -127,27 +241,71 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ - "!pygmentize model_with_header.json" + "%%writetemplate model_with_header.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example-header\n", + "spec:\n", + " annotations:\n", + " seldon.io/ambassador-header: 'location: london'\n", + " seldon.io/ambassador-service-name: example\n", + " name: header-model\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: single\n", + " replicas: 1" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-header created\r\n" + ] + } + ], "source": [ - "!kubectl create -f model_with_header.json" + "!kubectl create -f model_with_header.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-header-single-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-header-single-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-header -o jsonpath='{.items[0].metadata.name}')" ] @@ -161,9 +319,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.015688371980047133\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.052105223663186165]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\")\n", "print(r)" @@ -171,7 +350,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -180,9 +359,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['2']\n" + ] + } + ], "source": [ "print(default_count)\n", "assert(int(default_count[0]) == 2)" @@ -197,9 +384,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.6339240340443978\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.0925623293728051]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\",headers={\"location\":\"london\"})\n", "print(r)" @@ -207,7 +415,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ @@ -216,9 +424,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['1']\n" + ] + } + ], "source": [ "print(header_count)\n", "assert(int(header_count[0]) == 1)" @@ -226,20 +442,36 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 33, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f model.json" + "!kubectl delete -f model.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 34, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example-header\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f model_with_header.json" + "!kubectl delete -f model_with_header.yaml" ] }, { diff --git a/examples/ambassador/headers/model.json b/examples/ambassador/headers/model.json deleted file mode 100644 index d167518da9..0000000000 --- a/examples/ambassador/headers/model.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "example" - }, - "spec": { - "name": "production-model", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "single", - "replicas": 1 - } - ] - } -} - diff --git a/examples/ambassador/headers/model_with_header.json b/examples/ambassador/headers/model_with_header.json deleted file mode 100644 index 9ba2a49086..0000000000 --- a/examples/ambassador/headers/model_with_header.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "example-header" - }, - "spec": { - "name": "header-model", - "annotations": { - "seldon.io/ambassador-service-name":"example", - "seldon.io/ambassador-header":"location: london" - }, - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "single", - "replicas": 1 - } - ] - } -} - diff --git a/examples/ambassador/shadow/.gitignore b/examples/ambassador/shadow/.gitignore new file mode 100644 index 0000000000..2b158f424d --- /dev/null +++ b/examples/ambassador/shadow/.gitignore @@ -0,0 +1,2 @@ +model.yaml +shadow.yaml \ No newline at end of file diff --git a/examples/ambassador/shadow/ambassador_shadow.ipynb b/examples/ambassador/shadow/ambassador_shadow.ipynb index e5ee123e44..370884fbdb 100644 --- a/examples/ambassador/shadow/ambassador_shadow.ipynb +++ b/examples/ambassador/shadow/ambassador_shadow.ipynb @@ -22,9 +22,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] @@ -38,6 +46,42 @@ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "VERSION=!cat ../../../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -49,27 +93,67 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "!pygmentize model.json" + "%%writetemplate model.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example\n", + "spec:\n", + " name: production-model\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: default\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f model.json" + "!kubectl apply -f model.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"example-default-0-classifier\" successfully rolled out\r\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[0].metadata.name}')" ] @@ -83,7 +167,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -100,9 +184,30 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.6517840856894687\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.09407343218921792]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\")\n", "print(r)" @@ -119,27 +224,84 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ - "!pygmentize shadow.json" + "%%writetemplate shadow.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: example\n", + "spec:\n", + " name: shadow-model\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " terminationGracePeriodSeconds: 1\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: default\n", + " replicas: 1\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: shadow\n", + " replicas: 1\n", + " shadow: true\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example configured\r\n" + ] + } + ], "source": [ - "!kubectl apply -f shadow.json" + "!kubectl apply -f shadow.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"example-default-0-classifier\" successfully rolled out\n", + "Waiting for deployment \"example-shadow-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-shadow-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[0].metadata.name}')\n", "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example -o jsonpath='{.items[1].metadata.name}')" @@ -154,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -164,7 +326,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -173,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -182,9 +344,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['10']\n", + "['11']\n" + ] + } + ], "source": [ "print(shadow_count)\n", "print(default_count)\n", @@ -196,23 +367,24 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Return to previous model" + "## TearDown" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f model.json" - ] - }, - { - "cell_type": "markdown", + "execution_count": 21, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (NotFound): error when deleting \"model.yaml\": seldondeployments.machinelearning.seldon.io \"example\" not found\r\n" + ] + } + ], "source": [ - "## TearDown" + "!kubectl delete -f model.yaml" ] }, { @@ -220,9 +392,7 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "!kubectl delete -f model.json" - ] + "source": [] } ], "metadata": { diff --git a/examples/ambassador/shadow/model.json b/examples/ambassador/shadow/model.json deleted file mode 100644 index 230d0f84b6..0000000000 --- a/examples/ambassador/shadow/model.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "example" - }, - "spec": { - "name": "production-model", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "default", - "replicas": 1 - } - ] - } -} - diff --git a/examples/ambassador/shadow/shadow.json b/examples/ambassador/shadow/shadow.json deleted file mode 100644 index 28fcb0242b..0000000000 --- a/examples/ambassador/shadow/shadow.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "example" - }, - "spec": { - "name": "shadow-model", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ], - "terminationGracePeriodSeconds": 1 - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "default", - "replicas": 1 - }, - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/mock_classifier_rest:1.4", - "imagePullPolicy": "IfNotPresent", - "name": "classifier" - } - ] - }} - ], - "graph": - { - "children": [], - "name": "classifier", - "type": "MODEL", - "endpoint": { - "type": "REST" - }}, - "name": "shadow", - "replicas": 1, - "shadow": true - } - - ] - } -} - diff --git a/examples/explainers/imagenet/imagenet_explainer.yaml b/examples/explainers/imagenet/imagenet_explainer.yaml index 5ea0904fd2..9c0d0e2bcd 100644 --- a/examples/explainers/imagenet/imagenet_explainer.yaml +++ b/examples/explainers/imagenet/imagenet_explainer.yaml @@ -12,7 +12,7 @@ spec: - componentSpecs: - spec: containers: - - image: docker.io/seldonio/imagenet-transformer:0.1 + - image: docker.io/seldonio/imagenet-transformer:0.2 name: transformer graph: name: transformer diff --git a/examples/explainers/imagenet/resources/transformer/Makefile b/examples/explainers/imagenet/resources/transformer/Makefile index 3ae61345cf..b40982fdf0 100644 --- a/examples/explainers/imagenet/resources/transformer/Makefile +++ b/examples/explainers/imagenet/resources/transformer/Makefile @@ -1,11 +1,8 @@ -IMAGE_VERSION=0.1 +IMAGE_VERSION=0.2 IMAGE_NAME=docker.io/seldonio/imagenet-transformer -build_grpc: - s2i build -E environment_grpc . seldonio/seldon-core-s2i-python36:1.5.0-dev $(IMAGE_NAME):$(IMAGE_VERSION) - -build_rest: - s2i build -E environment_rest . seldonio/seldon-core-s2i-python36:1.5.0-dev $(IMAGE_NAME):$(IMAGE_VERSION) +build: + s2i build -E environment . seldonio/seldon-core-s2i-python36:1.5.0-dev $(IMAGE_NAME):$(IMAGE_VERSION) push_to_dockerhub: docker push $(IMAGE_NAME):$(IMAGE_VERSION) diff --git a/examples/explainers/imagenet/resources/transformer/environment_grpc b/examples/explainers/imagenet/resources/transformer/environment similarity index 83% rename from examples/explainers/imagenet/resources/transformer/environment_grpc rename to examples/explainers/imagenet/resources/transformer/environment index bc883fb1fd..3d26eb0709 100644 --- a/examples/explainers/imagenet/resources/transformer/environment_grpc +++ b/examples/explainers/imagenet/resources/transformer/environment @@ -1,5 +1,4 @@ MODEL_NAME=ImageNetTransformer -API_TYPE=GRPC SERVICE_TYPE=TRANSFORMER PERSISTENCE=0 diff --git a/examples/explainers/imagenet/resources/transformer/environment_rest b/examples/explainers/imagenet/resources/transformer/environment_rest deleted file mode 100644 index 624f0848bd..0000000000 --- a/examples/explainers/imagenet/resources/transformer/environment_rest +++ /dev/null @@ -1,5 +0,0 @@ -MODEL_NAME=ImageNetTransformer -API_TYPE=REST -SERVICE_TYPE=TRANSFORMER -PERSISTENCE=0 - diff --git a/examples/explainers/imagenet/resources/transformer/requirements.txt b/examples/explainers/imagenet/resources/transformer/requirements.txt index e69de29bb2..58f286f857 100644 --- a/examples/explainers/imagenet/resources/transformer/requirements.txt +++ b/examples/explainers/imagenet/resources/transformer/requirements.txt @@ -0,0 +1 @@ +tensorflow==1.15.4 diff --git a/examples/istio/canary_update/.gitignore b/examples/istio/canary_update/.gitignore deleted file mode 100644 index de2016048c..0000000000 --- a/examples/istio/canary_update/.gitignore +++ /dev/null @@ -1 +0,0 @@ -canary.py diff --git a/examples/istio/canary_update/README.md b/examples/istio/canary_update/README.md deleted file mode 100644 index f593fbd4b3..0000000000 --- a/examples/istio/canary_update/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Canary Roll Out using Seldon-Core and Istio - -This folder provides resources to illustrate how to do a canary roll out of one MNIST model to another using the canary pattern where a small amount of traffic is sent to the new model to validate it before sending all traffic to the new model. - -There is a [Jupyter Notebook](canary.ipynb) that provides a step by step demo. \ No newline at end of file diff --git a/examples/istio/canary_update/canary.ipynb b/examples/istio/canary_update/canary.ipynb deleted file mode 100644 index 5624928e3c..0000000000 --- a/examples/istio/canary_update/canary.ipynb +++ /dev/null @@ -1,427 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Canary Deployment with Seldon and Istio" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Running this example\n", - "\n", - "***This demo needs egress when running the load test to allow MNIST digits to be downloaded. If you want to run the load test then you will need to follow the docs on egress [here](https://istio.io/docs/tasks/traffic-management/egress/#calling-external-services-directly) if you run istio in a way that egress is blocked***" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup Cluster and Ingress\n", - "Use the setup notebook to [Setup Cluster](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Setup-Cluster) with [Istio Ingress](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Istio). Instructions [also online](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create namespace seldon" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup Istio\n", - "\n", - "Ensure you have istio installed. Follow their [docs](https://istio.io/latest/docs/)\n", - "\n", - "For this example we will create the default istio gateway for seldon which needs to be called `seldon-gateway`. You can supply your own gateway by adding to your SeldonDeployments resources the annotation `seldon.io/istio-gateway` with values the name of your istio gateway." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Create a gateway for our istio-ingress" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f ../../../notebooks/resources/seldon-gateway.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Ensure the istio ingress gatewaty is port-forwarded to localhost:8004\n", - "\n", - " * Istio: `kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8004:80`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "ISTIO_GATEWAY=\"localhost:8004\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "To view the istio traffic you can go to the istio grafana dashboard. In a separate terminal port-forward to it:\n", - "\n", - "See their docs [here](https://istio.io/latest/docs/tasks/observability/metrics/using-istio-dashboard/)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Serve Single Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from random import randint,random\n", - "import json\n", - "from matplotlib import pyplot as plt\n", - "import numpy as np\n", - "from tensorflow.examples.tutorials.mnist import input_data\n", - "from seldon_core.seldon_client import SeldonClient\n", - "import seldon_core\n", - "import time\n", - "\n", - "def gen_image(arr):\n", - " two_d = (np.reshape(arr, (28, 28)) * 255).astype(np.uint8)\n", - " plt.imshow(two_d,cmap=plt.cm.gray_r, interpolation='nearest')\n", - " return plt\n", - "\n", - "def download_mnist():\n", - " return input_data.read_data_sets(\"MNIST_data/\", one_hot = True)\n", - "\n", - "def predict_rest_mnist(mnist,deployment_name,namespace,istio_gateway):\n", - " sc = SeldonClient(deployment_name=deployment_name,namespace=namespace,gateway_endpoint=istio_gateway)\n", - " batch_xs, batch_ys = mnist.train.next_batch(1)\n", - " chosen=0\n", - " gen_image(batch_xs[chosen]).show()\n", - " data = batch_xs[chosen].reshape((1,784))\n", - " features = [\"X\"+str(i+1) for i in range (0,784)] \n", - " r = sc.predict(gateway=\"istio\",transport=\"rest\",shape=(1,784),data=data,payload_type='ndarray',names=features)\n", - " predictions = r.response\n", - " fpreds = [ '%.2f' % elem for elem in predictions[\"data\"][\"ndarray\"][0] ]\n", - " m = dict(zip(predictions[\"data\"][\"names\"],fpreds))\n", - " print(json.dumps(m,indent=2))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "mnist = download_mnist()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f mnist_v1.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mnist-classifier -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(60):\n", - " state=!kubectl get sdep mnist-classifier -o jsonpath='{.status.state}'\n", - " state=state[0]\n", - " print(state)\n", - " if state==\"Available\":\n", - " break\n", - " time.sleep(1)\n", - "assert(state==\"Available\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predict_rest_mnist(mnist,\"mnist-classifier\",\"seldon\",ISTIO_GATEWAY)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we will add a canary and split traffic 75% to 25% to it. This is done by adding a new predictor to the SeldonDeployment and specifying the traffic values." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize mnist_v2.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f mnist_v2.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mnist-classifier -o jsonpath='{.items[0].metadata.name}')\n", - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mnist-classifier -o jsonpath='{.items[1].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(60):\n", - " state=!kubectl get sdep mnist-classifier -o jsonpath='{.status.state}'\n", - " state=state[0]\n", - " print(state)\n", - " if state==\"Available\":\n", - " break\n", - " time.sleep(1)\n", - "assert(state==\"Available\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(1,100):\n", - " predict_rest_mnist(mnist,\"mnist-classifier\",\"seldon\",ISTIO_GATEWAY)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Check the traffic is split" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "default_count=!kubectl logs $(kubectl get pod -l seldon-app=mnist-classifier-sk-mnist-predictor -o jsonpath='{.items[0].metadata.name}') seldon-container-engine | grep \"Predictions called\" | wc -l" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "canary_count=!kubectl logs $(kubectl get pod -l seldon-app=mnist-classifier-tf-mnist-predictor -o jsonpath='{.items[0].metadata.name}') seldon-container-engine | grep \"Predictions called\" | wc -l" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "percentage_to_canary = float(canary_count[0])/float(default_count[0])\n", - "print(percentage_to_canary)\n", - "assert(percentage_to_canary > 0.1 and percentage_to_canary < 0.5)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "When you are happy the canary is ok you can promote to full traffic." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f mnist_v3.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mnist-classifier -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(60):\n", - " state=!kubectl get sdep mnist-classifier -o jsonpath='{.status.state}'\n", - " state=state[0]\n", - " print(state)\n", - " if state==\"Creating\":\n", - " break\n", - " time.sleep(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(60):\n", - " state=!kubectl get sdep mnist-classifier -o jsonpath='{.status.state}'\n", - " state=state[0]\n", - " print(state)\n", - " if state==\"Available\":\n", - " break\n", - " time.sleep(1)\n", - "assert(state==\"Available\")\n", - "time.sleep(3)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "predict_rest_mnist(mnist,\"mnist-classifier\",\"seldon\",ISTIO_GATEWAY)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl delete -f mnist_v3.json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "anaconda-cloud": {}, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.6.8" - }, - "varInspector": { - "cols": { - "lenName": 16, - "lenType": 16, - "lenVar": 40 - }, - "kernels_config": { - "python": { - "delete_cmd_postfix": "", - "delete_cmd_prefix": "del ", - "library": "var_list.py", - "varRefreshCmd": "print(var_dic_list())" - }, - "r": { - "delete_cmd_postfix": ") ", - "delete_cmd_prefix": "rm(", - "library": "var_list.r", - "varRefreshCmd": "cat(var_dic_list()) " - } - }, - "types_to_exclude": [ - "module", - "function", - "builtin_function_or_method", - "instance", - "_Feature" - ], - "window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/examples/istio/canary_update/istio_canary_v1.yaml b/examples/istio/canary_update/istio_canary_v1.yaml deleted file mode 100644 index fa7c143d28..0000000000 --- a/examples/istio/canary_update/istio_canary_v1.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: mnist-deployment - namespace: seldon -spec: - hosts: - - mnist-deployment-mnist-classifier - http: - - route: - - destination: - host: mnist-deployment-mnist-classifier - subset: v1 ---- -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: mnist-deployment - namespace: seldon -spec: - host: mnist-deployment-mnist-classifier - subsets: - - name: v1 - labels: - version: v1 - - name: v2 - labels: - version: v2 ---- - diff --git a/examples/istio/canary_update/istio_canary_v2.yaml b/examples/istio/canary_update/istio_canary_v2.yaml deleted file mode 100644 index 835dbd6473..0000000000 --- a/examples/istio/canary_update/istio_canary_v2.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: mnist-deployment - namespace: seldon -spec: - hosts: - - mnist-deployment-mnist-classifier - http: - - route: - - destination: - host: mnist-deployment-mnist-classifier - subset: v1 - weight: 70 - - destination: - host: mnist-deployment-mnist-classifier - subset: v2 - weight: 30 ---- -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: mnist-deployment - namespace: seldon -spec: - host: mnist-deployment-mnist-classifier - subsets: - - name: v1 - labels: - version: v1 - - name: v2 - labels: - version: v2 ---- - diff --git a/examples/istio/canary_update/istio_canary_v3.yaml b/examples/istio/canary_update/istio_canary_v3.yaml deleted file mode 100644 index 1e674b25e9..0000000000 --- a/examples/istio/canary_update/istio_canary_v3.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: mnist-deployment - namespace: seldon -spec: - hosts: - - mnist-deployment-mnist-classifier - http: - - route: - - destination: - host: mnist-deployment-mnist-classifier - subset: v2 ---- -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: mnist-deployment - namespace: seldon -spec: - host: mnist-deployment-mnist-classifier - subsets: - - name: v1 - labels: - version: v1 - - name: v2 - labels: - version: v2 ---- - diff --git a/examples/istio/canary_update/mnist_v1.json b/examples/istio/canary_update/mnist_v1.json deleted file mode 100644 index 245b4f28e8..0000000000 --- a/examples/istio/canary_update/mnist_v1.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "mnist-classifier" - }, - "spec": { - "annotations": { - "project_name": "Mnist classification" - }, - "name": "mnist-deployment", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/sk-example-mnist:0.3", - "imagePullPolicy": "IfNotPresent", - "name": "sk-mnist-classifier", - "resources": { - "requests": { - "memory": "1Mi" - } - } - } - ], - "terminationGracePeriodSeconds": 1 - } - }], - "graph": { - "children": [], - "name": "sk-mnist-classifier", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" - }, - "name": "sk-mnist-predictor", - "replicas": 1, - "labels":{ - "version":"v1" - } - } - ] - } -} diff --git a/examples/istio/canary_update/mnist_v2.json b/examples/istio/canary_update/mnist_v2.json deleted file mode 100644 index 596a370e00..0000000000 --- a/examples/istio/canary_update/mnist_v2.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "mnist-classifier" - }, - "spec": { - "annotations": { - "project_name": "Mnist classification" - }, - "name": "mnist-deployment", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/sk-example-mnist:0.3", - "imagePullPolicy": "IfNotPresent", - "name": "sk-mnist-classifier", - "resources": { - "requests": { - "memory": "1Mi" - } - } - } - ], - "terminationGracePeriodSeconds": 1 - } - }], - "graph": { - "children": [], - "name": "sk-mnist-classifier", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" - }, - "name": "sk-mnist-predictor", - "replicas": 1, - "svcOrchSpec": { - "env": [ - { - "name": "SELDON_LOG_LEVEL", - "value": "DEBUG" - } - ] - }, - "labels":{ - "version":"v1" - }, - "traffic": 75 - }, - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/tf-example-mnist:0.2", - "imagePullPolicy": "IfNotPresent", - "name": "tf-mnist-classifier", - "resources": { - "requests": { - "memory": "1Mi" - } - } - } - ], - "terminationGracePeriodSeconds": 1 - } - }], - "graph": { - "children": [], - "name": "tf-mnist-classifier", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" - }, - "name": "tf-mnist-predictor", - "replicas": 1, - "svcOrchSpec": { - "env": [ - { - "name": "SELDON_LOG_LEVEL", - "value": "DEBUG" - } - ] - }, - "labels":{ - "version":"v2" - }, - "traffic": 25 - } - ] - } -} diff --git a/examples/istio/canary_update/mnist_v3.json b/examples/istio/canary_update/mnist_v3.json deleted file mode 100644 index 4c258c3c31..0000000000 --- a/examples/istio/canary_update/mnist_v3.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "mnist-classifier" - }, - "spec": { - "annotations": { - "project_name": "Mnist classification" - }, - "name": "mnist-deployment", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/tf-example-mnist:0.2", - "imagePullPolicy": "IfNotPresent", - "name": "tf-mnist-classifier", - "resources": { - "requests": { - "memory": "1Mi" - } - } - } - ], - "terminationGracePeriodSeconds": 1 - } - }], - "graph": { - "children": [], - "name": "tf-mnist-classifier", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" - }, - "name": "tf-mnist-predictor", - "replicas": 1, - "labels":{ - "version":"v2" - } - } - - ] - } -} diff --git a/examples/istio/canary_update/setup_port_forwards.sh b/examples/istio/canary_update/setup_port_forwards.sh deleted file mode 100755 index ed2d20fc42..0000000000 --- a/examples/istio/canary_update/setup_port_forwards.sh +++ /dev/null @@ -1,8 +0,0 @@ -#Ambassador -kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8002:8080 & - -#Seldon Grafana -kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3001:3000 & - -#Istio grafana -kubectl -n istio-system port-forward $(kubectl -n istio-system get pod -l app=grafana -o jsonpath='{.items[0].metadata.name}') 3000:3000 & diff --git a/examples/istio/canary_update/sk-predictor.png b/examples/istio/canary_update/sk-predictor.png deleted file mode 100644 index 9f25b1de4f..0000000000 Binary files a/examples/istio/canary_update/sk-predictor.png and /dev/null differ diff --git a/examples/istio/canary_update/sk-predictor2.png b/examples/istio/canary_update/sk-predictor2.png deleted file mode 100644 index cf23982e74..0000000000 Binary files a/examples/istio/canary_update/sk-predictor2.png and /dev/null differ diff --git a/examples/istio/canary_update/tf-predictor.png b/examples/istio/canary_update/tf-predictor.png deleted file mode 100644 index c4c3010c3d..0000000000 Binary files a/examples/istio/canary_update/tf-predictor.png and /dev/null differ diff --git a/examples/istio/canary_update/tf-predictor2.png b/examples/istio/canary_update/tf-predictor2.png deleted file mode 100644 index 4b679a438e..0000000000 Binary files a/examples/istio/canary_update/tf-predictor2.png and /dev/null differ diff --git a/examples/istio/canary_update/visualizer.py b/examples/istio/canary_update/visualizer.py deleted file mode 100644 index 405bbb2750..0000000000 --- a/examples/istio/canary_update/visualizer.py +++ /dev/null @@ -1,36 +0,0 @@ -import graphviz -import json - -def _populate_graph(dot, root, suffix=''): - name = root.get("name") - id = name+suffix - if root.get("implementation"): - dot.node(id, label=name, shape="box", style="filled", color="lightgrey") - else: - dot.node(id, label=name, shape="box") - endpoint_type = root.get("endpoint",{}).get("type") - if endpoint_type is not None: - dot.node(id+'endpoint', label=endpoint_type) - dot.edge(id,id+'endpoint') - for child in root.get("children",[]): - child_id = _populate_graph(dot,child) - dot.edge(id, child_id) - return id - -def get_graph(filename,predictor=0): - deployment = json.load(open(filename,'r')) - predictors = deployment.get("spec").get("predictors") - dot = graphviz.Digraph() - - with dot.subgraph(name="cluster_0") as pdot: - graph = predictors[0].get("graph") - _populate_graph(pdot, graph, suffix='0') - pdot.attr(label="predictor") - - if len(predictors)>1: - with dot.subgraph(name="cluster_1") as cdot: - graph = predictors[1].get("graph") - _populate_graph(cdot, graph, suffix='1') - cdot.attr(label="canary") - - return dot diff --git a/examples/keda/.gitignore b/examples/keda/.gitignore new file mode 100644 index 0000000000..0b35c3b470 --- /dev/null +++ b/examples/keda/.gitignore @@ -0,0 +1 @@ +model_with_keda_prom.yaml \ No newline at end of file diff --git a/examples/keda/keda_prom_auto_scale.ipynb b/examples/keda/keda_prom_auto_scale.ipynb index f7f9398072..ceb0b2c5ea 100644 --- a/examples/keda/keda_prom_auto_scale.ipynb +++ b/examples/keda/keda_prom_auto_scale.ipynb @@ -1,27 +1,8 @@ { - "metadata": { - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7-final" - }, - "orig_nbformat": 2, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - } - }, - "nbformat": 4, - "nbformat_minor": 2, "cells": [ { + "cell_type": "markdown", + "metadata": {}, "source": [ "\n", "# Scale Seldon Deployments based on Prometheus Metrics.\n", @@ -31,22 +12,22 @@ "\n", "With the support of KEDA in Seldon, you can scale your seldon deployments with any scalers listed [here](https://keda.sh/docs/2.0/scalers/).\n", "In this example we will scale the seldon deployment with Prometheus metrics as an example." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Install Seldon Core\n", "\n", "Install Seldon Core as described in [docs](https://docs.seldon.io/projects/seldon-core/en/latest/workflow/install.html)\n", "\n", "Make sure add `--set istio.keda=true`" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Install Seldon Core Analytic\n", "\n", @@ -54,9 +35,7 @@ "Later we will use the Prometheus service installed to provide metrics in order to scale the Seldon models.\n", "\n", "Install Seldon Core Analytics as described in [docs](https://docs.seldon.io/projects/seldon-core/en/latest/analytics/analytics.html)" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -68,22 +47,22 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Install KEDA" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { - "source": [ - "!kubectl apply -f https://github.com/kedacore/keda/releases/download/v2.0.0-beta/keda-2.0.0-beta.yaml" - ], "cell_type": "code", + "execution_count": null, "metadata": { "tags": [] }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "!kubectl apply -f https://github.com/kedacore/keda/releases/download/v2.0.0-beta/keda-2.0.0-beta.yaml" + ] }, { "cell_type": "code", @@ -97,13 +76,15 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Create model with KEDA" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "To create a model with KEDA autoscaling you just need to add a KEDA spec refering in the Deployment, e.g.:\n", "```yaml\n", @@ -121,19 +102,61 @@ " query: rate(seldon_api_executor_client_requests_seconds_count{seldon_app=~\"seldon-model-example\"}[10s]\n", "```\n", "The full SeldonDeployment spec is shown below." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "tags": [] - }, + "metadata": {}, + "outputs": [], + "source": [ + "VERSION=!cat ../../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ - "!pygmentize model_with_keda_prom.yaml" + "%%writefile model_with_keda_prom.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: seldon-model\n", + "spec:\n", + " name: test-deployment\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:1.5.0-dev\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " resources:\n", + " requests:\n", + " cpu: '0.5'\n", + " kedaSpec:\n", + " pollingInterval: 15 # Optional. Default: 30 seconds\n", + " minReplicaCount: 1 # Optional. Default: 0\n", + " maxReplicaCount: 5 # Optional. Default: 100\n", + " triggers:\n", + " - type: prometheus\n", + " metadata:\n", + " # Required\n", + " serverAddress: http://seldon-core-analytics-prometheus-seldon.seldon-system.svc.cluster.local\n", + " metricName: access_frequency\n", + " threshold: '10'\n", + " query: rate(seldon_api_executor_client_requests_seconds_count{seldon_app=~\"seldon-model-example\"}[1m])\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " name: example\n" ] }, { @@ -159,37 +182,37 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Create Load" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "We label some nodes for the loadtester. We attempt the first two as for Kind the first node shown will be the master." - ], - "cell_type": "markdown", - "metadata": {} + ] }, { - "source": [ - "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust\n", - "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[1].metadata.name}') role=locust" - ], "cell_type": "code", + "execution_count": null, "metadata": { "tags": [] }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust\n", + "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[1].metadata.name}') role=locust" + ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "Before add loads to the model, there is only one replica" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -222,11 +245,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "After a few mins you should see the deployment scaled to 5 replicas" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -265,11 +288,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Remove Load" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -283,11 +306,11 @@ ] }, { + "cell_type": "markdown", + "metadata": {}, "source": [ "After 5-10 mins you should see the deployment replica number decrease to 1" - ], - "cell_type": "markdown", - "metadata": {} + ] }, { "cell_type": "code", @@ -318,5 +341,26 @@ "outputs": [], "source": [] } - ] -} \ No newline at end of file + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/keda/model_with_keda_prom.yaml b/examples/keda/model_with_keda_prom.yaml index 0c316c719b..27489d3b56 100644 --- a/examples/keda/model_with_keda_prom.yaml +++ b/examples/keda/model_with_keda_prom.yaml @@ -1,4 +1,3 @@ - apiVersion: machinelearning.seldon.io/v1 kind: SeldonDeployment metadata: @@ -9,7 +8,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/mock_classifier_rest:1.3 + - image: seldonio/mock_classifier:1.5.0-dev imagePullPolicy: IfNotPresent name: classifier resources: @@ -33,4 +32,4 @@ spec: type: REST name: classifier type: MODEL - name: example \ No newline at end of file + name: example diff --git a/examples/models/autoscaling/autoscaling_example.ipynb b/examples/models/autoscaling/autoscaling_example.ipynb index 1ea9853b0b..2955842494 100644 --- a/examples/models/autoscaling/autoscaling_example.ipynb +++ b/examples/models/autoscaling/autoscaling_example.ipynb @@ -35,18 +35,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] @@ -88,27 +104,81 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[34;01mapiVersion\u001b[39;49;00m: machinelearning.seldon.io/v1\r\n", + "\u001b[34;01mkind\u001b[39;49;00m: SeldonDeployment\r\n", + "\u001b[34;01mmetadata\u001b[39;49;00m:\r\n", + " \u001b[34;01mname\u001b[39;49;00m: seldon-model\r\n", + "\u001b[34;01mspec\u001b[39;49;00m:\r\n", + " \u001b[34;01mname\u001b[39;49;00m: test-deployment\r\n", + " \u001b[34;01mpredictors\u001b[39;49;00m:\r\n", + " - \u001b[34;01mcomponentSpecs\u001b[39;49;00m:\r\n", + " - \u001b[34;01mhpaSpec\u001b[39;49;00m:\r\n", + " \u001b[34;01mmaxReplicas\u001b[39;49;00m: 3\r\n", + " \u001b[34;01mmetrics\u001b[39;49;00m:\r\n", + " - \u001b[34;01mresource\u001b[39;49;00m:\r\n", + " \u001b[34;01mname\u001b[39;49;00m: cpu\r\n", + " \u001b[34;01mtargetAverageUtilization\u001b[39;49;00m: 10\r\n", + " \u001b[34;01mtype\u001b[39;49;00m: Resource\r\n", + " \u001b[34;01mminReplicas\u001b[39;49;00m: 1\r\n", + " \u001b[34;01mspec\u001b[39;49;00m:\r\n", + " \u001b[34;01mcontainers\u001b[39;49;00m:\r\n", + " - \u001b[34;01mimage\u001b[39;49;00m: seldonio/mock_classifier:1.5.0-dev\r\n", + " \u001b[34;01mimagePullPolicy\u001b[39;49;00m: IfNotPresent\r\n", + " \u001b[34;01mname\u001b[39;49;00m: classifier\r\n", + " \u001b[34;01mresources\u001b[39;49;00m:\r\n", + " \u001b[34;01mrequests\u001b[39;49;00m:\r\n", + " \u001b[34;01mcpu\u001b[39;49;00m: \u001b[33m'\u001b[39;49;00m\u001b[33m0.5\u001b[39;49;00m\u001b[33m'\u001b[39;49;00m\r\n", + " \u001b[34;01mterminationGracePeriodSeconds\u001b[39;49;00m: 1\r\n", + " \u001b[34;01mgraph\u001b[39;49;00m:\r\n", + " \u001b[34;01mchildren\u001b[39;49;00m: []\r\n", + " \u001b[34;01mname\u001b[39;49;00m: classifier\r\n", + " \u001b[34;01mtype\u001b[39;49;00m: MODEL\r\n", + " \u001b[34;01mname\u001b[39;49;00m: example\r\n" + ] + } + ], "source": [ "!pygmentize model_with_hpa.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/seldon-model created\r\n" + ] + } + ], "source": [ "!kubectl create -f model_with_hpa.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"seldon-model-example-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"seldon-model-example-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=seldon-model -o jsonpath='{.items[0].metadata.name}')" ] @@ -124,9 +194,18 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "error: 'role' already has a value (locust), and --overwrite is false\n", + "error: 'role' already has a value (locust), and --overwrite is false\n" + ] + } + ], "source": [ "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust\n", "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[1].metadata.name}') role=locust" @@ -134,9 +213,22 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: loadtester\r\n", + "LAST DEPLOYED: Sun Nov 1 13:13:47 2020\r\n", + "NAMESPACE: seldon\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], "source": [ "!helm install loadtester ../../../helm-charts/seldon-core-loadtesting \\\n", " --set locust.host=http://seldon-model-example:8000 \\\n", @@ -158,9 +250,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "1\n", + "3\n" + ] + } + ], "source": [ "import json\n", "import time\n", @@ -200,27 +317,68 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"loadtester\" uninstalled\r\n" + ] + } + ], "source": [ "!helm delete loadtester -n seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\r\n", + "pod/ambassador-6747c68887-2rddl 1/1 Running 0 22h\r\n", + "pod/jaeger-5cb557b89d-khfb8 1/1 Running 0 22h\r\n", + "pod/jaeger-operator-67777ffc99-m25fp 1/1 Running 0 22h\r\n", + "pod/locust-master-1-6sbss 1/1 Running 0 125m\r\n", + "pod/locust-slave-1-nlwgv 1/1 Running 0 125m\r\n", + "pod/seldon-model-example-0-classifier-7cf4bd7485-fvn7f 2/2 Running 0 126m\r\n", + "pod/seldon-model-example-0-classifier-7cf4bd7485-jlsjg 2/2 Running 0 124m\r\n", + "pod/seldon-model-example-0-classifier-7cf4bd7485-p9j4w 0/2 Pending 0 124m\r\n", + "\r\n", + "NAME READY UP-TO-DATE AVAILABLE AGE\r\n", + "deployment.apps/ambassador 1/1 1 1 22h\r\n", + "deployment.apps/jaeger 1/1 1 1 22h\r\n", + "deployment.apps/jaeger-operator 1/1 1 1 22h\r\n", + "deployment.apps/seldon-model-example-0-classifier 2/3 3 2 126m\r\n", + "\r\n", + "NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE\r\n", + "horizontalpodautoscaler.autoscaling/seldon-model-example-0-classifier Deployment/seldon-model-example-0-classifier 29%/10% 1 3 3 126m\r\n" + ] + } + ], "source": [ "!kubectl get pods,deployments,hpa" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"seldon-model\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f model_with_hpa.yaml" ] diff --git a/examples/models/autoscaling/model_with_hpa.yaml b/examples/models/autoscaling/model_with_hpa.yaml index 938307ae1a..22f031d87e 100644 --- a/examples/models/autoscaling/model_with_hpa.yaml +++ b/examples/models/autoscaling/model_with_hpa.yaml @@ -16,7 +16,7 @@ spec: minReplicas: 1 spec: containers: - - image: seldonio/mock_classifier_rest:1.3 + - image: seldonio/mock_classifier:1.5.0-dev imagePullPolicy: IfNotPresent name: classifier resources: @@ -25,8 +25,6 @@ spec: terminationGracePeriodSeconds: 1 graph: children: [] - endpoint: - type: REST name: classifier type: MODEL name: example diff --git a/examples/models/custom_metrics/Makefile b/examples/models/custom_metrics/Makefile index 9b1a915368..8d85a698bc 100644 --- a/examples/models/custom_metrics/Makefile +++ b/examples/models/custom_metrics/Makefile @@ -1,28 +1,16 @@ -VERSION=0.1 +VERSION=0.2 IMAGE_BASE=seldonio/model-with-metrics -PYTHON_WRAPPER=0.19-SNAPSHOT +PYTHON_WRAPPER:= $(shell cat ../../../version.txt) -build_rest: - s2i build -E environment_rest . seldonio/seldon-core-s2i-python36:${PYTHON_WRAPPER} ${IMAGE_BASE}_rest:${VERSION} +build: + s2i build -E environment . seldonio/seldon-core-s2i-python36:${PYTHON_WRAPPER} ${IMAGE_BASE}:${VERSION} -push_rest: - docker push ${IMAGE_BASE}_rest:${VERSION} +push: + docker push ${IMAGE_BASE}:${VERSION} -build_grpc: - s2i build -E environment_grpc . seldonio/seldon-core-s2i-python36:${PYTHON_WRAPPER} ${IMAGE_BASE}_grpc:${VERSION} +run_local: + export PREDICTIVE_UNIT_SERVICE_PORT=9000 && TRACING=1 JAEGER_AGENT_HOST=localhost JAEGER_AGENT_PORT=6831 JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 seldon-core-microservice --service-type MODEL ModelWithMetrics -push_grpc: - docker push ${IMAGE_BASE}_grpc:${VERSION} +kind_load: build + kind load -v 3 docker-image ${IMAGE_BASE}:${VERSION} -run_rest_local: - export PREDICTIVE_UNIT_SERVICE_PORT=9000 && TRACING=1 JAEGER_AGENT_HOST=localhost JAEGER_AGENT_PORT=6831 JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 seldon-core-microservice --service-type MODEL ModelWithMetrics REST - -run_grpc_local: - export PREDICTIVE_UNIT_SERVICE_PORT=9000 && TRACING=1 JAEGER_AGENT_HOST=localhost JAEGER_AGENT_PORT=6831 JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 seldon-core-microservice --service-type MODEL ModelWithMetrics GRPC - - -kind_load_rest: build_rest - kind load -v 3 docker-image ${IMAGE_BASE}_rest:${VERSION} - -kind_load_grpc: build_grpc - kind load -v 3 docker-image ${IMAGE_BASE}_grpc:${VERSION} diff --git a/examples/models/custom_metrics/customMetrics.ipynb b/examples/models/custom_metrics/customMetrics.ipynb index 54733d42eb..ebfa3b219d 100644 --- a/examples/models/custom_metrics/customMetrics.ipynb +++ b/examples/models/custom_metrics/customMetrics.ipynb @@ -33,34 +33,62 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: seldon-core-analytics\r\n", + "LAST DEPLOYED: Sun Nov 8 10:24:13 2020\r\n", + "NAMESPACE: seldon-system\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n" + ] + } + ], "source": [ "!helm install seldon-core-analytics ../../../helm-charts/seldon-core-analytics -n seldon-system --wait" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -72,32 +100,74 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Custom Metrics with a REST model" + "## Custom Metrics" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[94mapiVersion\u001b[39;49;00m: machinelearning.seldon.io/v1\r\n", + "\u001b[94mkind\u001b[39;49;00m: SeldonDeployment\r\n", + "\u001b[94mmetadata\u001b[39;49;00m:\r\n", + " \u001b[94mname\u001b[39;49;00m: seldon-model\r\n", + "\u001b[94mspec\u001b[39;49;00m:\r\n", + " \u001b[94mname\u001b[39;49;00m: test-deployment\r\n", + " \u001b[94mpredictors\u001b[39;49;00m:\r\n", + " - \u001b[94mcomponentSpecs\u001b[39;49;00m:\r\n", + " - \u001b[94mspec\u001b[39;49;00m:\r\n", + " \u001b[94mcontainers\u001b[39;49;00m:\r\n", + " - \u001b[94mimage\u001b[39;49;00m: seldonio/model-with-metrics:0.2\r\n", + " \u001b[94mname\u001b[39;49;00m: classifier\r\n", + " \u001b[94mgraph\u001b[39;49;00m:\r\n", + " \u001b[94mchildren\u001b[39;49;00m: []\r\n", + " \u001b[94mname\u001b[39;49;00m: classifier\r\n", + " \u001b[94mtype\u001b[39;49;00m: MODEL\r\n", + " \u001b[94mname\u001b[39;49;00m: example\r\n", + " \u001b[94mreplicas\u001b[39;49;00m: 1\r\n" + ] + } + ], "source": [ - "!pygmentize model_rest.yaml" + "!pygmentize model.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/seldon-model created\r\n" + ] + } + ], "source": [ - "!kubectl create -f model_rest.yaml" + "!kubectl create -f model.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"seldon-model-example-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"seldon-model-example-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=seldon-model \\\n", " -o jsonpath='{.items[0].metadata.name}')" @@ -105,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -114,9 +184,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['t:0', 't:1', 't:2'], 'ndarray': [[1.0, 2.0, 5.0]]}, 'meta': {'metrics': [{'key': 'mycounter', 'type': 'COUNTER', 'value': 1}, {'key': 'mygauge', 'type': 'GAUGE', 'value': 100}, {'key': 'mytimer', 'type': 'TIMER', 'value': 20.2}]}}\n" + ] + } + ], "source": [ "response=json.loads(responseRaw[0])\n", "print(response)\n", @@ -125,19 +203,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting so metrics can be scraped\n" + ] + } + ], "source": [ "print(\"Waiting so metrics can be scraped\")\n", - "time.sleep(10)" + "time.sleep(30)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting get-metrics.sh\n" + ] + } + ], "source": [ "%%writefile get-metrics.sh\n", "\n", @@ -147,7 +241,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -156,18 +250,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'{\"status\":\"success\",\"data\":{\"resultType\":\"vector\",\"result\":[{\"metric\":{\"__name__\":\"mycounter_total\",\"app\":\"seldon-model-example-0-classifier\",\"app_kubernetes_io_managed_by\":\"seldon-core\",\"deployment_name\":\"seldon-model\",\"fluentd\":\"true\",\"image_name\":\"seldonio/model-with-metrics\",\"image_version\":\"0.2\",\"instance\":\"10.244.1.109:6000\",\"job\":\"kubernetes-pods\",\"kubernetes_namespace\":\"seldon\",\"kubernetes_pod_name\":\"seldon-model-example-0-classifier-869f44f779-547dj\",\"method\":\"predict\",\"model_image\":\"seldonio/model-with-metrics\",\"model_name\":\"classifier\",\"model_version\":\"0.2\",\"pod_template_hash\":\"869f44f779\",\"predictor_name\":\"example\",\"predictor_version\":\"example\",\"seldon_app\":\"seldon-model-example\",\"seldon_app_svc\":\"seldon-model-example-classifier\",\"seldon_deployment_id\":\"seldon-model\",\"seldon_deployment_name\":\"seldon-model\",\"seldon_io_default\":\"true\",\"seldon_io_model\":\"true\",\"version\":\"example\",\"worker_id\":\"40\"},\"value\":[1604831146.35,\"1\"]}]}}'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "responseRaw[0]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'mycounter_total', 'app': 'seldon-model-example-0-classifier', 'app_kubernetes_io_managed_by': 'seldon-core', 'deployment_name': 'seldon-model', 'fluentd': 'true', 'image_name': 'seldonio/model-with-metrics', 'image_version': '0.2', 'instance': '10.244.1.109:6000', 'job': 'kubernetes-pods', 'kubernetes_namespace': 'seldon', 'kubernetes_pod_name': 'seldon-model-example-0-classifier-869f44f779-547dj', 'method': 'predict', 'model_image': 'seldonio/model-with-metrics', 'model_name': 'classifier', 'model_version': '0.2', 'pod_template_hash': '869f44f779', 'predictor_name': 'example', 'predictor_version': 'example', 'seldon_app': 'seldon-model-example', 'seldon_app_svc': 'seldon-model-example-classifier', 'seldon_deployment_id': 'seldon-model', 'seldon_deployment_name': 'seldon-model', 'seldon_io_default': 'true', 'seldon_io_model': 'true', 'version': 'example', 'worker_id': '40'}, 'value': [1604831146.35, '1']}]}}\n" + ] + } + ], "source": [ "response=json.loads(responseRaw[0])\n", "print(response)\n", @@ -176,51 +289,7 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl delete -f model_rest.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Custom Metrics with a GRPC model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize model_grpc.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl create -f model_grpc.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=seldon-model \\\n", - " -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -232,9 +301,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {'metrics': [{'key': 'mycounter', 'value': 1}, {'key': 'mygauge', 'type': 'GAUGE', 'value': 100}, {'key': 'mytimer', 'type': 'TIMER', 'value': 20.2}]}, 'data': {'names': ['t:0', 't:1', 't:2'], 'ndarray': [[1, 2, 5]]}}\n" + ] + } + ], "source": [ "response=json.loads(\"\".join(responseRaw))\n", "print(response)\n", @@ -243,17 +320,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting so metrics can be scraped\n" + ] + } + ], "source": [ "print(\"Waiting so metrics can be scraped\")\n", - "time.sleep(10)" + "time.sleep(30)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -262,33 +347,64 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'mycounter_total', 'app': 'seldon-model-example-0-classifier', 'app_kubernetes_io_managed_by': 'seldon-core', 'deployment_name': 'seldon-model', 'fluentd': 'true', 'image_name': 'seldonio/model-with-metrics', 'image_version': '0.2', 'instance': '10.244.1.73:6000', 'job': 'kubernetes-pods', 'kubernetes_namespace': 'seldon', 'kubernetes_pod_name': 'seldon-model-example-0-classifier-869f44f779-hk8p2', 'method': 'predict', 'model_image': 'seldonio/model-with-metrics', 'model_name': 'classifier', 'model_version': '0.2', 'pod_template_hash': '869f44f779', 'predictor_name': 'example', 'predictor_version': 'example', 'seldon_app': 'seldon-model-example', 'seldon_app_svc': 'seldon-model-example-classifier', 'seldon_deployment_id': 'seldon-model', 'seldon_deployment_name': 'seldon-model', 'seldon_io_default': 'true', 'seldon_io_model': 'true', 'version': 'example', 'worker_id': '42'}, 'value': [1604666136.309, '1']}, {'metric': {'__name__': 'mycounter_total', 'app': 'seldon-model-example-0-classifier', 'app_kubernetes_io_managed_by': 'seldon-core', 'deployment_name': 'seldon-model', 'fluentd': 'true', 'image_name': 'seldonio/model-with-metrics', 'image_version': '0.2', 'instance': '10.244.1.73:6000', 'job': 'kubernetes-pods', 'kubernetes_namespace': 'seldon', 'kubernetes_pod_name': 'seldon-model-example-0-classifier-869f44f779-hk8p2', 'method': 'predict', 'model_image': 'seldonio/model-with-metrics', 'model_name': 'classifier', 'model_version': '0.2', 'pod_template_hash': '869f44f779', 'predictor_name': 'example', 'predictor_version': 'example', 'seldon_app': 'seldon-model-example', 'seldon_app_svc': 'seldon-model-example-classifier', 'seldon_deployment_id': 'seldon-model', 'seldon_deployment_name': 'seldon-model', 'seldon_io_default': 'true', 'seldon_io_model': 'true', 'version': 'example', 'worker_id': '31'}, 'value': [1604666136.309, '1']}]}}\n" + ] + } + ], "source": [ "response=json.loads(responseRaw[0])\n", "print(response)\n", "assert(response['data'][\"result\"][0][\"metric\"][\"__name__\"]=='mycounter_total')\n", - "assert(response['data'][\"result\"][0][\"metric\"][\"image_name\"]=='seldonio/model-with-metrics_grpc')" + "assert(response['data'][\"result\"][0][\"metric\"][\"image_name\"]=='seldonio/model-with-metrics')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"seldon-model\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f model_grpc.yaml" + "!kubectl delete -f model.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"seldon-core-analytics\" uninstalled\r\n" + ] + } + ], "source": [ "!helm delete seldon-core-analytics --namespace seldon-system" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -307,7 +423,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/models/custom_metrics/environment_rest b/examples/models/custom_metrics/environment similarity index 100% rename from examples/models/custom_metrics/environment_rest rename to examples/models/custom_metrics/environment diff --git a/examples/models/custom_metrics/environment_grpc b/examples/models/custom_metrics/environment_grpc deleted file mode 100644 index 5cb5826fbe..0000000000 --- a/examples/models/custom_metrics/environment_grpc +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=ModelWithMetrics -API_TYPE=GRPC -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/examples/models/custom_metrics/model_rest.yaml b/examples/models/custom_metrics/model.yaml similarity index 78% rename from examples/models/custom_metrics/model_rest.yaml rename to examples/models/custom_metrics/model.yaml index 79befb365f..af5daf0b0e 100644 --- a/examples/models/custom_metrics/model_rest.yaml +++ b/examples/models/custom_metrics/model.yaml @@ -8,12 +8,10 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/model-with-metrics_rest:0.1 + - image: seldonio/model-with-metrics:0.2 name: classifier graph: children: [] - endpoint: - type: REST name: classifier type: MODEL name: example diff --git a/examples/models/custom_metrics/model_grpc.yaml b/examples/models/custom_metrics/model_grpc.yaml deleted file mode 100644 index fc56d920be..0000000000 --- a/examples/models/custom_metrics/model_grpc.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1 -kind: SeldonDeployment -metadata: - name: seldon-model -spec: - name: test-deployment - predictors: - - componentSpecs: - - spec: - containers: - - image: seldonio/model-with-metrics_grpc:0.1 - name: classifier - graph: - children: [] - endpoint: - type: GRPC - name: classifier - type: MODEL - name: example - replicas: 1 diff --git a/examples/models/deep_mnist/.gitignore b/examples/models/deep_mnist/.gitignore deleted file mode 100644 index 47e40d4b46..0000000000 --- a/examples/models/deep_mnist/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -model/ -MNIST_data/ \ No newline at end of file diff --git a/examples/models/deep_mnist/.s2i/environment b/examples/models/deep_mnist/.s2i/environment deleted file mode 100644 index d5a1d6d66a..0000000000 --- a/examples/models/deep_mnist/.s2i/environment +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=DeepMnist -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/examples/models/deep_mnist/DeepMnist.py b/examples/models/deep_mnist/DeepMnist.py deleted file mode 100644 index d0ba299373..0000000000 --- a/examples/models/deep_mnist/DeepMnist.py +++ /dev/null @@ -1,27 +0,0 @@ -import tensorflow as tf -import numpy as np -import os - -class DeepMnist(object): - def __init__(self): - self.loaded = False - self.class_names = ["class:{}".format(str(i)) for i in range(10)] - - def load(self): - print("Loading model",os.getpid()) - self.sess = tf.Session() - saver = tf.train.import_meta_graph("model/deep_mnist_model.meta") - saver.restore(self.sess,tf.train.latest_checkpoint("./model/")) - graph = tf.get_default_graph() - self.x = graph.get_tensor_by_name("x:0") - self.y = graph.get_tensor_by_name("y:0") - self.loaded = True - print("Loaded model") - - def predict(self,X,feature_names): - if not self.loaded: - self.load() - predictions = self.sess.run(self.y,feed_dict={self.x:X}) - return predictions.astype(np.float64) - - diff --git a/examples/models/deep_mnist/Makefile b/examples/models/deep_mnist/Makefile deleted file mode 100644 index 8bde2522c7..0000000000 --- a/examples/models/deep_mnist/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -IMAGE_NAME=seldonio/tf-example-mnist -IMAGE_VERSION=0.2 - -build_image: - s2i build . seldonio/seldon-core-s2i-python3:1.5.0-dev ${IMAGE_NAME}:${IMAGE_VERSION} - -push_image: - docker push $(IMAGE_NAME):$(IMAGE_VERSION) - -kind_load: - kind load -v 3 docker-image ${IMAGE_NAME}:${IMAGE_VERSION} - -.PHONY: train -train: - mkdir -p model - python create_model.py - -.PHONY: clean -clean: - rm -rf model - rm -rf MNIST_data diff --git a/examples/models/deep_mnist/create_model.py b/examples/models/deep_mnist/create_model.py deleted file mode 100644 index 2ea52b6aa5..0000000000 --- a/examples/models/deep_mnist/create_model.py +++ /dev/null @@ -1,36 +0,0 @@ -from tensorflow.examples.tutorials.mnist import input_data -mnist = input_data.read_data_sets("MNIST_data/", one_hot = True) -import tensorflow as tf - -if __name__ == '__main__': - - x = tf.placeholder(tf.float32, [None,784], name="x") - - W = tf.Variable(tf.zeros([784,10])) - b = tf.Variable(tf.zeros([10])) - - y = tf.nn.softmax(tf.matmul(x,W) + b, name="y") - - y_ = tf.placeholder(tf.float32, [None, 10]) - - - cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1])) - - train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy) - - init = tf.initialize_all_variables() - - sess = tf.Session() - sess.run(init) - - for i in range(1000): - batch_xs, batch_ys = mnist.train.next_batch(100) - sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) - - correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1)) - accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) - print(sess.run(accuracy, feed_dict = {x: mnist.test.images, y_:mnist.test.labels})) - - saver = tf.train.Saver() - - saver.save(sess, "model/deep_mnist_model") diff --git a/examples/models/deep_mnist/deep_mnist.json b/examples/models/deep_mnist/deep_mnist.json deleted file mode 100644 index 350a4c85bc..0000000000 --- a/examples/models/deep_mnist/deep_mnist.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "deep-mnist" - }, - "spec": { - "annotations": { - "project_name": "Tensorflow MNIST", - "deployment_version": "v1" - }, - "name": "deep-mnist", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "deep-mnist:0.1", - "imagePullPolicy": "IfNotPresent", - "name": "classifier", - "resources": { - "requests": { - "memory": "1Mi" - } - } - } - ], - "terminationGracePeriodSeconds": 20 - } - }], - "graph": { - "children": [], - "name": "classifier", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" - }, - "name": "single-model", - "replicas": 1, - "annotations": { - "predictor_version" : "v1" - } - } - ] - } -} diff --git a/examples/models/deep_mnist/requirements.txt b/examples/models/deep_mnist/requirements.txt deleted file mode 100644 index 68394a62c4..0000000000 --- a/examples/models/deep_mnist/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -tensorflow>=1.12.0,<2.0 diff --git a/examples/models/mean_classifier/Makefile b/examples/models/mean_classifier/Makefile index 6d248da9f5..90ac8c1df1 100644 --- a/examples/models/mean_classifier/Makefile +++ b/examples/models/mean_classifier/Makefile @@ -4,28 +4,17 @@ IMAGE_NAME_BASE=mock_classifier IMAGE_BASE=seldonio/${IMAGE_NAME_BASE} -build_%: - s2i build -E environment_$* . seldonio/seldon-core-s2i-python37-ubi8:${VERSION} ${IMAGE_BASE}_$*:${VERSION} +build: + s2i build -E environment . seldonio/seldon-core-s2i-python37-ubi8:${VERSION} ${IMAGE_BASE}:${VERSION} -push_%: - docker push ${IMAGE_BASE}_$*:${VERSION} +push: + docker push ${IMAGE_BASE}:${VERSION} -.PHONY: build_all -build_all: build_rest build_grpc -.PHONY: push_all -push_all:push_rest push_grpc +run_local: + export PREDICTIVE_UNIT_HTTP_SERVICE_PORT=9001 && export PREDICTIVE_UNIT_GRPC_SERVICE_PORT=5001 && export TRACING=1 && export JAEGER_AGENT_HOST=localhost && export JAEGER_AGENT_PORT=6831 && export JAEGER_SAMPLER_TYPE=const && export JAEGER_SAMPLER_PARAM=1 && export SELDON_DEBUG=0 && seldon-core-microservice --service-type MODEL MeanClassifier -run_rest_local: - export PREDICTIVE_UNIT_SERVICE_PORT=9000 && TRACING=1 JAEGER_AGENT_HOST=localhost JAEGER_AGENT_PORT=6831 JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 seldon-core-microservice --service-type MODEL MeanClassifier REST - -run_grpc_local: - export PREDICTIVE_UNIT_SERVICE_PORT=9000 && TRACING=1 JAEGER_AGENT_HOST=localhost JAEGER_AGENT_PORT=6831 JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 seldon-core-microservice --service-type MODEL MeanClassifier GRPC - - -kind_load_%: - kind load -v 3 docker-image ${IMAGE_BASE}_$*:${VERSION} -.PHONY: kind_load_all -kind_load_all: kind_load_rest kind_load_grpc +kind_load: build + kind load -v 3 docker-image ${IMAGE_BASE}:${VERSION} # # Redhat @@ -35,13 +24,12 @@ kind_load_all: kind_load_rest kind_load_grpc scan_rest=ospid-c90fdfe6-d054-4598-baa8-7f7aac3ed63a # no groc at present scan_grpc= -redhat-image-scan-%: - docker pull ${IMAGE_BASE}_$*:${VERSION} +scan= +redhat-image-scan: + docker pull ${IMAGE_BASE}:${VERSION} source ~/.config/seldon/seldon-core/redhat-image-passwords.sh && \ - echo $${rh_password_mock_model_$*} | docker login -u unused scan.connect.redhat.com --password-stdin - docker tag ${IMAGE_BASE}_$*:${VERSION} scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} - docker push scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} + echo $${rh_password_mock_model} | docker login -u unused scan.connect.redhat.com --password-stdin + docker tag ${IMAGE_BASE}:${VERSION} scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} + docker push scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} -.PHONY: redhat-image-scan-all -redhat-image-scan: redhat-image-scan-rest diff --git a/examples/models/mean_classifier/environment_grpc b/examples/models/mean_classifier/environment similarity index 80% rename from examples/models/mean_classifier/environment_grpc rename to examples/models/mean_classifier/environment index c14140e5e9..255fef9c24 100644 --- a/examples/models/mean_classifier/environment_grpc +++ b/examples/models/mean_classifier/environment @@ -1,4 +1,3 @@ MODEL_NAME=MeanClassifier -API_TYPE=GRPC SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/examples/models/mean_classifier/environment_rest b/examples/models/mean_classifier/environment_rest deleted file mode 100644 index e20631ca80..0000000000 --- a/examples/models/mean_classifier/environment_rest +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=MeanClassifier -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/examples/models/metadata/example-grpc.yaml b/examples/models/metadata/example-grpc.yaml index 5742ae8e78..f103b0630d 100644 --- a/examples/models/metadata/example-grpc.yaml +++ b/examples/models/metadata/example-grpc.yaml @@ -10,7 +10,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/metadata-generic-node_grpc:0.4 + - image: seldonio/metadata-generic-node:0.4 name: node-one env: - name: MODEL_METADATA @@ -29,7 +29,7 @@ spec: names: [one-output] custom: author: seldon-dev - - image: seldonio/metadata-generic-node_grpc:0.4 + - image: seldonio/metadata-generic-node:0.4 name: node-two env: diff --git a/examples/models/metadata/graph-metadata/combiner.yaml b/examples/models/metadata/graph-metadata/combiner.yaml index d5ebb2e3f3..50c837dbaf 100644 --- a/examples/models/metadata/graph-metadata/combiner.yaml +++ b/examples/models/metadata/graph-metadata/combiner.yaml @@ -9,14 +9,14 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-combiner env: - name: MODEL_METADATA value: | --- name: node-combiner - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor @@ -32,14 +32,14 @@ spec: schema: names: [combiner-output] shape: [ 1 ] - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-one env: - name: MODEL_METADATA value: | --- name: node-one - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor @@ -51,14 +51,14 @@ spec: schema: names: [ c1 ] shape: [ 1 ] - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-two env: - name: MODEL_METADATA value: | --- name: node-two - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor diff --git a/examples/models/metadata/graph-metadata/input-transformer.yaml b/examples/models/metadata/graph-metadata/input-transformer.yaml index 2ba8a85b95..40e82cde2f 100644 --- a/examples/models/metadata/graph-metadata/input-transformer.yaml +++ b/examples/models/metadata/graph-metadata/input-transformer.yaml @@ -9,14 +9,14 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-input-transformer env: - name: MODEL_METADATA value: | --- name: node-input-transformer - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor @@ -28,14 +28,14 @@ spec: schema: names: [transformer-output] shape: [ 1 ] - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node env: - name: MODEL_METADATA value: | --- name: node - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor diff --git a/examples/models/metadata/graph-metadata/output-transformer.yaml b/examples/models/metadata/graph-metadata/output-transformer.yaml index 2e1ffcda22..1df90d629f 100644 --- a/examples/models/metadata/graph-metadata/output-transformer.yaml +++ b/examples/models/metadata/graph-metadata/output-transformer.yaml @@ -9,14 +9,14 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-output-transformer env: - name: MODEL_METADATA value: | --- name: node-output-transformer - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor @@ -28,14 +28,14 @@ spec: schema: names: [transformer-output] shape: [ 1 ] - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node env: - name: MODEL_METADATA value: | --- name: node - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor diff --git a/examples/models/metadata/graph-metadata/router.yaml b/examples/models/metadata/graph-metadata/router.yaml index af7eea88ec..0f956146e5 100644 --- a/examples/models/metadata/graph-metadata/router.yaml +++ b/examples/models/metadata/graph-metadata/router.yaml @@ -9,16 +9,16 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-router - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-one env: - name: MODEL_METADATA value: | --- name: node-one - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor @@ -30,14 +30,14 @@ spec: schema: names: [ node-output ] shape: [ 1 ] - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-two env: - name: MODEL_METADATA value: | --- name: node-two - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor diff --git a/examples/models/metadata/graph-metadata/single.yaml b/examples/models/metadata/graph-metadata/single.yaml index b9540cdaf1..bebc1356f6 100644 --- a/examples/models/metadata/graph-metadata/single.yaml +++ b/examples/models/metadata/graph-metadata/single.yaml @@ -9,14 +9,14 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: model env: - name: MODEL_METADATA value: | --- name: single-node - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor diff --git a/examples/models/metadata/graph-metadata/two-levels.yaml b/examples/models/metadata/graph-metadata/two-levels.yaml index e938ca6643..a56048c698 100644 --- a/examples/models/metadata/graph-metadata/two-levels.yaml +++ b/examples/models/metadata/graph-metadata/two-levels.yaml @@ -9,14 +9,14 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-one env: - name: MODEL_METADATA value: | --- name: node-one - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor @@ -28,14 +28,14 @@ spec: schema: names: [ a3 ] shape: [ 1 ] - - image: seldonio/metadata-generic-node_rest:0.3 + - image: seldonio/metadata-generic-node:0.4 name: node-two env: - name: MODEL_METADATA value: | --- name: node-two - versions: [ generic-node/v0.3 ] + versions: [ generic-node/v0.4 ] platform: seldon inputs: - messagetype: tensor diff --git a/examples/models/metadata/graph_metadata.ipynb b/examples/models/metadata/graph_metadata.ipynb index 63a23224d7..c5e679daa8 100644 --- a/examples/models/metadata/graph_metadata.ipynb +++ b/examples/models/metadata/graph_metadata.ipynb @@ -28,7 +28,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Error from server (AlreadyExists): namespaces \"seldon\" already exists\n" + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" ] } ], @@ -45,7 +45,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Context \"kind-kind\" modified.\n" + "Context \"kind-kind\" modified.\r\n" ] } ], @@ -173,14 +173,14 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: model\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: single-node\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -209,7 +209,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/graph-metadata-single created\n" + "seldondeployment.machinelearning.seldon.io/graph-metadata-single created\r\n" ] } ], @@ -219,15 +219,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Waiting for deployment \"graph-metadata-single-example-0-model\" rollout to finish: 0 of 1 updated replicas are available...\n", - "deployment \"graph-metadata-single-example-0-model\" successfully rolled out\n" + "deployment \"graph-metadata-single-example-0-model\" successfully rolled out\r\n" ] } ], @@ -246,7 +245,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -265,23 +264,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Failed request with status code 404\n" - ] - }, { "data": { "text/plain": [ "{'name': 'example',\n", " 'models': {'model': {'name': 'single-node',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['node-input'], 'shape': [1]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", @@ -292,7 +284,7 @@ " 'schema': {'names': ['node-output'], 'shape': [1]}}]}" ] }, - "execution_count": 8, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -306,7 +298,7 @@ " \"model\": {\n", " \"name\": \"single-node\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"node-input\"], \"shape\": [1]}}\n", " ],\n", @@ -337,22 +329,23 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'inputs': [{'messagetype': 'tensor',\n", + "{'custom': {},\n", + " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['node-input'], 'shape': [1]}}],\n", " 'name': 'single-node',\n", " 'outputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['node-output'], 'shape': [1]}}],\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3']}" + " 'versions': ['generic-node/v0.4']}" ] }, - "execution_count": 9, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } @@ -362,9 +355,10 @@ "meta = getWithRetry(\"http://localhost:8003/seldon/seldon/graph-metadata-single/api/v1.0/metadata/model\")\n", "\n", "assert meta == {\n", + " \"custom\": {},\n", " \"name\": \"single-node\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [{\n", " \"messagetype\": \"tensor\", \"schema\": {\"names\": [\"node-input\"], \"shape\": [1]},\n", " }],\n", @@ -376,6 +370,23 @@ "meta" ] }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"graph-metadata-single\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f graph-metadata/single.yaml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -393,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -417,14 +428,14 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-one\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-one\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -436,14 +447,14 @@ " schema:\n", " names: [ a3 ]\n", " shape: [ 1 ] \n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-two\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-two\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -468,14 +479,14 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/graph-metadata-two-levels created\n" + "seldondeployment.machinelearning.seldon.io/graph-metadata-two-levels created\r\n" ] } ], @@ -485,7 +496,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -503,30 +514,23 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 20, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Failed request with status code 404\n" - ] - }, { "data": { "text/plain": [ "{'name': 'example',\n", " 'models': {'node-one': {'name': 'node-one',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['a1', 'a2'], 'shape': [2]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['a3'], 'shape': [1]}}]},\n", " 'node-two': {'name': 'node-two',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['a3'], 'shape': [1]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", @@ -537,7 +541,7 @@ " 'schema': {'names': ['b1', 'b2'], 'shape': [2]}}]}" ] }, - "execution_count": 13, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -552,7 +556,7 @@ " \"node-one\": {\n", " \"name\": \"node-one\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"a1\", \"a2\"], \"shape\": [2]}}\n", " ],\n", @@ -563,7 +567,7 @@ " \"node-two\": {\n", " \"name\": \"node-two\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"a3\"], \"shape\": [1]}}\n", " ],\n", @@ -583,6 +587,23 @@ "meta" ] }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"graph-metadata-two-levels\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f graph-metadata/two-levels.yaml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -600,7 +621,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -624,14 +645,14 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-combiner\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-combiner\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -647,14 +668,14 @@ " schema:\n", " names: [combiner-output]\n", " shape: [ 1 ] \n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-one\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-one\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -666,14 +687,14 @@ " schema:\n", " names: [ c1 ]\n", " shape: [ 1 ] \n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-two\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-two\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -701,14 +722,14 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/graph-metadata-combiner created\n" + "seldondeployment.machinelearning.seldon.io/graph-metadata-combiner created\r\n" ] } ], @@ -718,7 +739,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 23, "metadata": {}, "outputs": [ { @@ -736,23 +757,16 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 24, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Failed request with status code 404\n" - ] - }, { "data": { "text/plain": [ "{'name': 'example',\n", " 'models': {'node-combiner': {'name': 'node-combiner',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['c1'], 'shape': [1]}},\n", " {'messagetype': 'tensor', 'schema': {'names': ['c2'], 'shape': [1]}}],\n", @@ -760,14 +774,14 @@ " 'schema': {'names': ['combiner-output'], 'shape': [1]}}]},\n", " 'node-one': {'name': 'node-one',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['a', 'b'], 'shape': [2]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['c1'], 'shape': [1]}}]},\n", " 'node-two': {'name': 'node-two',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['a', 'b'], 'shape': [2]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", @@ -778,7 +792,7 @@ " 'schema': {'names': ['combiner-output'], 'shape': [1]}}]}" ] }, - "execution_count": 17, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } @@ -793,7 +807,7 @@ " \"node-combiner\": {\n", " \"name\": \"node-combiner\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"c1\"], \"shape\": [1]}},\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"c2\"], \"shape\": [1]}},\n", @@ -805,7 +819,7 @@ " \"node-one\": {\n", " \"name\": \"node-one\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"a\", \"b\"], \"shape\": [2]}},\n", " ],\n", @@ -816,7 +830,7 @@ " \"node-two\": {\n", " \"name\": \"node-two\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"a\", \"b\"], \"shape\": [2]}},\n", " ],\n", @@ -836,6 +850,23 @@ "meta" ] }, + { + "cell_type": "code", + "execution_count": 39, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"graph-metadata-combiner\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f graph-metadata/combiner.yaml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -853,7 +884,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -877,16 +908,16 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-router\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-one\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-one\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -898,14 +929,14 @@ " schema:\n", " names: [ node-output ] \n", " shape: [ 1 ]\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-two\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-two\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -933,14 +964,14 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/graph-metadata-router created\n" + "seldondeployment.machinelearning.seldon.io/graph-metadata-router created\r\n" ] } ], @@ -950,7 +981,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -968,34 +999,27 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 29, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Failed request with status code 404\n" - ] - }, { "data": { "text/plain": [ "{'name': 'example',\n", " 'models': {'node-one': {'name': 'node-one',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['a', 'b'], 'shape': [2]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['node-output'], 'shape': [1]}}]},\n", - " 'node-router': {'name': 'seldonio/metadata-generic-node_rest',\n", - " 'versions': ['0.3'],\n", + " 'node-router': {'name': 'seldonio/metadata-generic-node',\n", + " 'versions': ['0.4'],\n", " 'inputs': [],\n", " 'outputs': []},\n", " 'node-two': {'name': 'node-two',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['a', 'b'], 'shape': [2]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", @@ -1006,7 +1030,7 @@ " 'schema': {'names': ['node-output'], 'shape': [1]}}]}" ] }, - "execution_count": 21, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } @@ -1019,15 +1043,15 @@ " \"name\": \"example\",\n", " \"models\": {\n", " 'node-router': {\n", - " 'name': 'seldonio/metadata-generic-node_rest',\n", - " 'versions': ['0.3'],\n", + " 'name': 'seldonio/metadata-generic-node',\n", + " 'versions': ['0.4'],\n", " 'inputs': [],\n", " 'outputs': [],\n", " },\n", " \"node-one\": {\n", " \"name\": \"node-one\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"a\", \"b\"], \"shape\": [2]}}\n", " ],\n", @@ -1038,7 +1062,7 @@ " \"node-two\": {\n", " \"name\": \"node-two\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [\n", " {\"messagetype\": \"tensor\", \"schema\": {\"names\": [\"a\", \"b\"], \"shape\": [2]}}\n", " ],\n", @@ -1058,6 +1082,23 @@ "meta" ] }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"graph-metadata-router\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f graph-metadata/router.yaml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1073,7 +1114,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -1097,14 +1138,14 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-input-transformer\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-input-transformer\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -1116,14 +1157,14 @@ " schema:\n", " names: [transformer-output] \n", " shape: [ 1 ] \n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -1148,14 +1189,14 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/graph-metadata-input created\n" + "seldondeployment.machinelearning.seldon.io/graph-metadata-input created\r\n" ] } ], @@ -1165,7 +1206,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 32, "metadata": {}, "outputs": [ { @@ -1183,30 +1224,23 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 33, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Failed request with status code 404\n" - ] - }, { "data": { "text/plain": [ "{'name': 'example',\n", " 'models': {'node': {'name': 'node',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['transformer-output'], 'shape': [1]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['node-output'], 'shape': [1]}}]},\n", " 'node-input-transformer': {'name': 'node-input-transformer',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['transformer-input'], 'shape': [1]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", @@ -1217,7 +1251,7 @@ " 'schema': {'names': ['node-output'], 'shape': [1]}}]}" ] }, - "execution_count": 25, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -1232,7 +1266,7 @@ " \"node-input-transformer\": {\n", " \"name\": \"node-input-transformer\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [{\n", " \"messagetype\": \"tensor\", \"schema\": {\"names\": [\"transformer-input\"], \"shape\": [1]},\n", " }],\n", @@ -1243,7 +1277,7 @@ " \"node\": {\n", " \"name\": \"node\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [{\n", " \"messagetype\": \"tensor\", \"schema\": {\"names\": [\"transformer-output\"], \"shape\": [1]},\n", " }],\n", @@ -1263,6 +1297,23 @@ "meta" ] }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"graph-metadata-input\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f graph-metadata/input-transformer.yaml" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1278,7 +1329,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -1302,14 +1353,14 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-output-transformer\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node-output-transformer\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -1321,14 +1372,14 @@ " schema:\n", " names: [transformer-output] \n", " shape: [ 1 ] \n", - " - image: seldonio/metadata-generic-node_rest:0.3\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node\n", " env:\n", " - name: MODEL_METADATA\n", " value: |\n", " ---\n", " name: node\n", - " versions: [ generic-node/v0.3 ]\n", + " versions: [ generic-node/v0.4 ]\n", " platform: seldon\n", " inputs:\n", " - messagetype: tensor\n", @@ -1353,14 +1404,14 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 42, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/graph-metadata-output created\n" + "seldondeployment.machinelearning.seldon.io/graph-metadata-output configured\r\n" ] } ], @@ -1370,15 +1421,14 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 43, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Waiting for deployment \"graph-metadata-output-example-0-node-output-transformer-node\" rollout to finish: 0 of 1 updated replicas are available...\n", - "deployment \"graph-metadata-output-example-0-node-output-transformer-node\" successfully rolled out\n" + "deployment \"graph-metadata-output-example-0-node-output-transformer-node\" successfully rolled out\r\n" ] } ], @@ -1388,30 +1438,23 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 44, "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Failed request with status code 404\n" - ] - }, { "data": { "text/plain": [ "{'name': 'example',\n", " 'models': {'node': {'name': 'node',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['node-input'], 'shape': [1]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['transformer-input'], 'shape': [1]}}]},\n", " 'node-output-transformer': {'name': 'node-output-transformer',\n", " 'platform': 'seldon',\n", - " 'versions': ['generic-node/v0.3'],\n", + " 'versions': ['generic-node/v0.4'],\n", " 'inputs': [{'messagetype': 'tensor',\n", " 'schema': {'names': ['transformer-input'], 'shape': [1]}}],\n", " 'outputs': [{'messagetype': 'tensor',\n", @@ -1422,7 +1465,7 @@ " 'schema': {'names': ['transformer-output'], 'shape': [1]}}]}" ] }, - "execution_count": 29, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -1437,7 +1480,7 @@ " \"node-output-transformer\": {\n", " \"name\": \"node-output-transformer\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [{\n", " \"messagetype\": \"tensor\", \"schema\": {\"names\": [\"transformer-input\"], \"shape\": [1]},\n", " }],\n", @@ -1448,7 +1491,7 @@ " \"node\": {\n", " \"name\": \"node\",\n", " \"platform\": \"seldon\",\n", - " \"versions\": [\"generic-node/v0.3\"],\n", + " \"versions\": [\"generic-node/v0.4\"],\n", " \"inputs\": [{\n", " \"messagetype\": \"tensor\", \"schema\": {\"names\": [\"node-input\"], \"shape\": [1]},\n", " }],\n", @@ -1468,33 +1511,21 @@ "meta" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Cleanup resources" - ] - }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 45, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io \"graph-metadata-combiner\" deleted\n", - "seldondeployment.machinelearning.seldon.io \"graph-metadata-input\" deleted\n", - "seldondeployment.machinelearning.seldon.io \"graph-metadata-output\" deleted\n", - "seldondeployment.machinelearning.seldon.io \"graph-metadata-router\" deleted\n", - "seldondeployment.machinelearning.seldon.io \"graph-metadata-single\" deleted\n", - "seldondeployment.machinelearning.seldon.io \"graph-metadata-two-levels\" deleted\n" + "seldondeployment.machinelearning.seldon.io \"graph-metadata-output\" deleted\r\n" ] } ], "source": [ - "!kubectl delete -f graph-metadata/" + "!kubectl delete -f graph-metadata/output-transformer.yaml" ] } ], @@ -1514,7 +1545,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.6" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/models/metadata/metadata.ipynb b/examples/models/metadata/metadata.ipynb index 6a88a78d79..ba6122839c 100644 --- a/examples/models/metadata/metadata.ipynb +++ b/examples/models/metadata/metadata.ipynb @@ -29,7 +29,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Error from server (AlreadyExists): namespaces \"seldon\" already exists\n" + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" ] } ], @@ -46,7 +46,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Context \"kind-seldon\" modified.\n" + "Context \"kind-kind\" modified.\r\n" ] } ], @@ -215,7 +215,7 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/model-with-metadata:0.4\n", + " - image: seldonio/model-with-metadata:0.5\n", " name: my-model\n", " env:\n", " - name: SELDON_LOG_LEVEL\n", @@ -244,7 +244,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/seldon-model-init-metadata created\n" + "seldondeployment.machinelearning.seldon.io/seldon-model-init-metadata created\r\n" ] } ], @@ -417,7 +417,7 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/model-with-metadata:0.4\n", + " - image: seldonio/model-with-metadata:0.5\n", " name: my-model\n", " env:\n", " - name: SELDON_LOG_LEVEL\n", @@ -452,7 +452,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/seldon-model-environ-metadata created\n" + "seldondeployment.machinelearning.seldon.io/seldon-model-environ-metadata created\r\n" ] } ], @@ -487,7 +487,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -502,7 +502,7 @@ " 'versions': ['my-model-version-01']}" ] }, - "execution_count": 14, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -535,7 +535,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -554,7 +554,7 @@ " 'graphoutputs': [{'messagetype': 'tensor', 'schema': {'shape': [1]}}]}" ] }, - "execution_count": 15, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" } @@ -598,7 +598,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -622,7 +622,7 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/model-with-metadata:0.3\n", + " - image: seldonio/model-with-metadata:0.5\n", " name: my-model\n", " env:\n", " - name: SELDON_LOG_LEVEL\n", @@ -642,14 +642,14 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/seldon-model-invalid-environ-metadata created\n" + "seldondeployment.machinelearning.seldon.io/seldon-model-invalid-environ-metadata created\r\n" ] } ], @@ -659,7 +659,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -684,7 +684,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -696,7 +696,7 @@ " 'status': 1}}" ] }, - "execution_count": 19, + "execution_count": 18, "metadata": {}, "output_type": "execute_result" } @@ -725,7 +725,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -752,7 +752,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -769,6 +769,13 @@ "%%bash\n", "kubectl delete -f model-metadata/" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -787,7 +794,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/models/metadata/metadata_grpc.ipynb b/examples/models/metadata/metadata_grpc.ipynb index 00e2dbfd84..aa08427285 100644 --- a/examples/models/metadata/metadata_grpc.ipynb +++ b/examples/models/metadata/metadata_grpc.ipynb @@ -28,7 +28,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Error from server (AlreadyExists): namespaces \"seldon\" already exists\n" + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" ] } ], @@ -45,7 +45,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Context \"kind-seldon\" modified.\n" + "Context \"kind-kind\" modified.\r\n" ] } ], @@ -91,7 +91,7 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/metadata-generic-node_grpc:0.4\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-one\n", " env: \n", " - name: MODEL_METADATA\n", @@ -110,7 +110,7 @@ " names: [one-output]\n", " custom:\n", " author: seldon-dev\n", - " - image: seldonio/metadata-generic-node_grpc:0.4\n", + " - image: seldonio/metadata-generic-node:0.4\n", " name: node-two\n", " env:\n", " \n", @@ -150,7 +150,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io/graph-metadata-grpc created\n" + "seldondeployment.machinelearning.seldon.io/graph-metadata-grpc created\r\n" ] } ], @@ -179,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -229,7 +229,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -237,7 +237,7 @@ "output_type": "stream", "text": [ "{\n", - " \"name\": \"node-two\",\n", + " \"name\": \"node-one\",\n", " \"versions\": [\n", " \"generic-node/v0.4\"\n", " ],\n", @@ -247,7 +247,7 @@ " \"messagetype\": \"tensor\",\n", " \"schema\": {\n", " \"names\": [\n", - " \"two-input\"\n", + " \"one-input\"\n", " ]\n", " }\n", " }\n", @@ -257,7 +257,7 @@ " \"messagetype\": \"tensor\",\n", " \"schema\": {\n", " \"names\": [\n", - " \"two-output\"\n", + " \"one-output\"\n", " ]\n", " }\n", " }\n", @@ -279,7 +279,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -314,7 +314,10 @@ " ]\n", " }\n", " }\n", - " ]\n", + " ],\n", + " \"custom\": {\n", + " \"author\": \"seldon-dev\"\n", + " }\n", " },\n", " \"node-two\": {\n", " \"name\": \"node-two\",\n", @@ -341,7 +344,10 @@ " ]\n", " }\n", " }\n", - " ]\n", + " ],\n", + " \"custom\": {\n", + " \"author\": \"seldon-dev\"\n", + " }\n", " }\n", " },\n", " \"inputs\": [\n", @@ -377,20 +383,27 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "seldondeployment.machinelearning.seldon.io \"graph-metadata-grpc\" deleted\n" + "seldondeployment.machinelearning.seldon.io \"graph-metadata-grpc\" deleted\r\n" ] } ], "source": [ "!kubectl delete -f example-grpc.yaml" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -409,7 +422,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.5" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/examples/models/metadata/model-metadata/environ-metadata.yaml b/examples/models/metadata/model-metadata/environ-metadata.yaml index bf7d15ab47..ee497ccddd 100644 --- a/examples/models/metadata/model-metadata/environ-metadata.yaml +++ b/examples/models/metadata/model-metadata/environ-metadata.yaml @@ -9,7 +9,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/model-with-metadata:0.4 + - image: seldonio/model-with-metadata:0.5 name: my-model env: - name: SELDON_LOG_LEVEL diff --git a/examples/models/metadata/model-metadata/init-metadata.yaml b/examples/models/metadata/model-metadata/init-metadata.yaml index 923d5b7ea0..b3de724845 100644 --- a/examples/models/metadata/model-metadata/init-metadata.yaml +++ b/examples/models/metadata/model-metadata/init-metadata.yaml @@ -9,7 +9,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/model-with-metadata:0.4 + - image: seldonio/model-with-metadata:0.5 name: my-model env: - name: SELDON_LOG_LEVEL diff --git a/examples/models/metadata/model-metadata/invalid-environ-metadata.yaml b/examples/models/metadata/model-metadata/invalid-environ-metadata.yaml index a6f83b18ed..45ac0ef08f 100644 --- a/examples/models/metadata/model-metadata/invalid-environ-metadata.yaml +++ b/examples/models/metadata/model-metadata/invalid-environ-metadata.yaml @@ -9,7 +9,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/model-with-metadata:0.3 + - image: seldonio/model-with-metadata:0.5 name: my-model env: - name: SELDON_LOG_LEVEL diff --git a/examples/models/metadata/models/generic-node/Makefile b/examples/models/metadata/models/generic-node/Makefile index d0c39b2621..8b8de2247f 100644 --- a/examples/models/metadata/models/generic-node/Makefile +++ b/examples/models/metadata/models/generic-node/Makefile @@ -1,29 +1,20 @@ VERSION=0.4 IMAGE_BASE=seldonio/metadata-generic-node -S2I_IMAGE_VERSION=1.2.3-dev +S2I_IMAGE_VERSION:= $(shell cat ../../../../../version.txt) + KIND_NAME ?= kind -build_%: +build: s2i build \ - -E environment_$* \ + -E environment \ . \ seldonio/seldon-core-s2i-python37:${S2I_IMAGE_VERSION} \ - ${IMAGE_BASE}_$*:${VERSION} - -push_%: - docker push ${IMAGE_BASE}_$*:${VERSION} - -kind_load_%: - kind load -v 3 docker-image ${IMAGE_BASE}_$*:${VERSION} --name ${KIND_NAME} - -.PHONY: build_all -build_all: build_rest build_grpc + ${IMAGE_BASE}:${VERSION} -.PHONY: push_all -push_all: push_rest push_grpc +push: + docker push ${IMAGE_BASE}:${VERSION} -.PHONY: kind_load_all -kind_load_all: kind_load_rest kind_load_grpc +kind_load: build + kind load -v 3 docker-image ${IMAGE_BASE}:${VERSION} --name ${KIND_NAME} -kind_image_install: build_all kind_load_all diff --git a/examples/models/metadata/models/generic-node/environment_rest b/examples/models/metadata/models/generic-node/environment similarity index 83% rename from examples/models/metadata/models/generic-node/environment_rest rename to examples/models/metadata/models/generic-node/environment index 73373d2b23..6a441c7280 100644 --- a/examples/models/metadata/models/generic-node/environment_rest +++ b/examples/models/metadata/models/generic-node/environment @@ -1,5 +1,4 @@ MODEL_NAME=Node -API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0 SELDON_LOG_LEVEL=DEBUG diff --git a/examples/models/metadata/models/generic-node/environment_grpc b/examples/models/metadata/models/generic-node/environment_grpc deleted file mode 100644 index 4752b30193..0000000000 --- a/examples/models/metadata/models/generic-node/environment_grpc +++ /dev/null @@ -1,5 +0,0 @@ -MODEL_NAME=Node -API_TYPE=GRPC -SERVICE_TYPE=MODEL -PERSISTENCE=0 -SELDON_LOG_LEVEL=DEBUG diff --git a/examples/models/metadata/models/init-metadata/Makefile b/examples/models/metadata/models/init-metadata/Makefile index cfa41ec205..913e6f3844 100644 --- a/examples/models/metadata/models/init-metadata/Makefile +++ b/examples/models/metadata/models/init-metadata/Makefile @@ -1,6 +1,6 @@ -VERSION=0.4 +VERSION=0.5 IMAGE_BASE=seldonio/model-with-metadata -S2I_IMAGE_VERSION=1.2.3-dev +S2I_IMAGE_VERSION:= $(shell cat ../../../../../version.txt) KIND_NAME ?= kind diff --git a/examples/models/metadata/models/init-metadata/environment b/examples/models/metadata/models/init-metadata/environment index 4e5ffe6f2c..b91522e558 100644 --- a/examples/models/metadata/models/init-metadata/environment +++ b/examples/models/metadata/models/init-metadata/environment @@ -1,4 +1,3 @@ MODEL_NAME=Model -API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/examples/models/payload_logging/.gitignore b/examples/models/payload_logging/.gitignore index 2b2db2c6bb..a198d8cfaf 100644 --- a/examples/models/payload_logging/.gitignore +++ b/examples/models/payload_logging/.gitignore @@ -1 +1,2 @@ -payload_logging.py \ No newline at end of file +payload_logging.py +model_logger.yaml diff --git a/examples/models/payload_logging/model_logger.yaml b/examples/models/payload_logging/model_logger.yaml deleted file mode 100644 index 445efdc881..0000000000 --- a/examples/models/payload_logging/model_logger.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1 -kind: SeldonDeployment -metadata: - name: model-logs -spec: - name: model-logs - predictors: - - componentSpecs: - - spec: - containers: - - image: seldonio/mock_classifier_rest:1.3 - name: classifier - imagePullPolicy: Always - graph: - children: [] - endpoint: - type: REST - name: classifier - type: MODEL - logger: - url: http://logger.seldon/ - mode: all - name: logging - replicas: 1 diff --git a/examples/models/payload_logging/payload_logging.ipynb b/examples/models/payload_logging/payload_logging.ipynb index dda9203bf9..b8ec641fbb 100644 --- a/examples/models/payload_logging/payload_logging.ipynb +++ b/examples/models/payload_logging/payload_logging.ipynb @@ -36,22 +36,74 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "VERSION=!cat ../../../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -63,27 +115,87 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[94mapiVersion\u001b[39;49;00m: apps/v1\r\n", + "\u001b[94mkind\u001b[39;49;00m: Deployment\r\n", + "\u001b[94mmetadata\u001b[39;49;00m:\r\n", + " \u001b[94mname\u001b[39;49;00m: logger\r\n", + "\u001b[94mspec\u001b[39;49;00m:\r\n", + " \u001b[94mselector\u001b[39;49;00m:\r\n", + " \u001b[94mmatchLabels\u001b[39;49;00m:\r\n", + " \u001b[94mrun\u001b[39;49;00m: logger\r\n", + " \u001b[94mreplicas\u001b[39;49;00m: 1\r\n", + " \u001b[94mtemplate\u001b[39;49;00m:\r\n", + " \u001b[94mmetadata\u001b[39;49;00m:\r\n", + " \u001b[94mlabels\u001b[39;49;00m:\r\n", + " \u001b[94mrun\u001b[39;49;00m: logger\r\n", + " \u001b[94mspec\u001b[39;49;00m:\r\n", + " \u001b[94mcontainers\u001b[39;49;00m:\r\n", + " - \u001b[94mname\u001b[39;49;00m: logger\r\n", + " \u001b[94mimage\u001b[39;49;00m: mendhak/http-https-echo\r\n", + " \u001b[94mports\u001b[39;49;00m:\r\n", + " - \u001b[94mcontainerPort\u001b[39;49;00m: 80\r\n", + "\u001b[04m\u001b[36m---\u001b[39;49;00m\r\n", + "\u001b[94mapiVersion\u001b[39;49;00m: v1\r\n", + "\u001b[94mkind\u001b[39;49;00m: Service\r\n", + "\u001b[94mmetadata\u001b[39;49;00m:\r\n", + " \u001b[94mname\u001b[39;49;00m: logger\r\n", + " \u001b[94mlabels\u001b[39;49;00m:\r\n", + " \u001b[94mrun\u001b[39;49;00m: logger\r\n", + "\u001b[94mspec\u001b[39;49;00m:\r\n", + " \u001b[94mports\u001b[39;49;00m:\r\n", + " - \u001b[94mport\u001b[39;49;00m: 80\r\n", + " \u001b[94mtargetPort\u001b[39;49;00m: 80\r\n", + " \u001b[94mprotocol\u001b[39;49;00m: TCP\r\n", + " \u001b[94mselector\u001b[39;49;00m:\r\n", + " \u001b[94mrun\u001b[39;49;00m: logger\r\n", + "\r\n", + " \r\n" + ] + } + ], "source": [ "!pygmentize message-dumper.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment.apps/logger created\n", + "service/logger created\n" + ] + } + ], "source": [ "!kubectl apply -f message-dumper.yaml -n seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"logger\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"logger\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/logger" ] @@ -97,27 +209,67 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ - "!pygmentize model_logger.yaml" + "%%writetemplate model_logger.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: model-logs\n", + "spec:\n", + " name: model-logs\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " name: classifier\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: REST\n", + " name: classifier\n", + " type: MODEL\n", + " logger:\n", + " url: http://logger.seldon/\n", + " mode: all\n", + " name: logging\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/model-logs created\r\n" + ] + } + ], "source": [ "!kubectl apply -f model_logger.yaml -n seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"model-logs-logging-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"model-logs-logging-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=model-logs -o jsonpath='{.items[0].metadata.name}')" ] @@ -131,9 +283,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['{\"data\":{\"names\":[\"proba\"],\"ndarray\":[[0.43782349911420193]]},\"meta\":{}}']\n" + ] + } + ], "source": [ "res=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", " -X POST http://localhost:8003/seldon/seldon/model-logs/api/v1.0/predictions \\\n", @@ -153,22 +313,137 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-----------------\r\n", + "{\r\n", + " \"path\": \"/\",\r\n", + " \"headers\": {\r\n", + " \"host\": \"logger.seldon\",\r\n", + " \"user-agent\": \"Go-http-client/1.1\",\r\n", + " \"content-length\": \"39\",\r\n", + " \"ce-endpoint\": \"logging\",\r\n", + " \"ce-id\": \"e276a0c3-a522-4499-b509-5e98e06e96fe\",\r\n", + " \"ce-inferenceservicename\": \"model-logs\",\r\n", + " \"ce-modelid\": \"classifier\",\r\n", + " \"ce-namespace\": \"seldon\",\r\n", + " \"ce-requestid\": \"33fb419e-8d6b-44e0-a084-8bd4bd4dbf7b\",\r\n", + " \"ce-source\": \"http://:8000/\",\r\n", + " \"ce-specversion\": \"1.0\",\r\n", + " \"ce-time\": \"2020-11-06T09:35:09.171644169Z\",\r\n", + " \"ce-traceparent\": \"00-9c7cc210d352f7b68be6422c1b4b78f4-a7e8a2a38709bbc9-00\",\r\n", + " \"ce-type\": \"io.seldon.serving.inference.request\",\r\n", + " \"content-type\": \"application/json\",\r\n", + " \"traceparent\": \"00-9c7cc210d352f7b68be6422c1b4b78f4-b49edd5ad25552ae-00\",\r\n", + " \"accept-encoding\": \"gzip\"\r\n", + " },\r\n", + " \"method\": \"POST\",\r\n", + " \"body\": \"{\\\"data\\\": {\\\"ndarray\\\":[[1.0, 2.0, 5.0]]}}\",\r\n", + " \"fresh\": false,\r\n", + " \"hostname\": \"logger.seldon\",\r\n", + " \"ip\": \"::ffff:10.244.1.65\",\r\n", + " \"ips\": [],\r\n", + " \"protocol\": \"http\",\r\n", + " \"query\": {},\r\n", + " \"subdomains\": [],\r\n", + " \"xhr\": false,\r\n", + " \"os\": {\r\n", + " \"hostname\": \"logger-766f99b9b7-mqtql\"\r\n", + " },\r\n", + " \"connection\": {},\r\n", + " \"json\": {\r\n", + " \"data\": {\r\n", + " \"ndarray\": [\r\n", + " [\r\n", + " 1,\r\n", + " 2,\r\n", + " 5\r\n", + " ]\r\n", + " ]\r\n", + " }\r\n", + " }\r\n", + "}\r\n", + "::ffff:10.244.1.65 - - [06/Nov/2020:09:35:09 +0000] \"POST / HTTP/1.1\" 200 1220 \"-\" \"Go-http-client/1.1\"\r\n", + "-----------------\r\n", + "{\r\n", + " \"path\": \"/\",\r\n", + " \"headers\": {\r\n", + " \"host\": \"logger.seldon\",\r\n", + " \"user-agent\": \"Go-http-client/1.1\",\r\n", + " \"content-length\": \"73\",\r\n", + " \"ce-endpoint\": \"logging\",\r\n", + " \"ce-id\": \"68345c32-d144-4e21-a840-55e7e809c002\",\r\n", + " \"ce-inferenceservicename\": \"model-logs\",\r\n", + " \"ce-modelid\": \"classifier\",\r\n", + " \"ce-namespace\": \"seldon\",\r\n", + " \"ce-requestid\": \"33fb419e-8d6b-44e0-a084-8bd4bd4dbf7b\",\r\n", + " \"ce-source\": \"http://:8000/\",\r\n", + " \"ce-specversion\": \"1.0\",\r\n", + " \"ce-time\": \"2020-11-06T09:35:09.180317759Z\",\r\n", + " \"ce-traceparent\": \"00-cbb2fa5d83dbc42f2f8e9f8957b5c121-c15418121da2e992-00\",\r\n", + " \"ce-type\": \"io.seldon.serving.inference.response\",\r\n", + " \"content-type\": \"application/json\",\r\n", + " \"traceparent\": \"00-cbb2fa5d83dbc42f2f8e9f8957b5c121-ce0a53c967ee8077-00\",\r\n", + " \"accept-encoding\": \"gzip\"\r\n", + " },\r\n", + " \"method\": \"POST\",\r\n", + " \"body\": \"{\\\"data\\\":{\\\"names\\\":[\\\"proba\\\"],\\\"ndarray\\\":[[0.43782349911420193]]},\\\"meta\\\":{}}\\n\",\r\n", + " \"fresh\": false,\r\n", + " \"hostname\": \"logger.seldon\",\r\n", + " \"ip\": \"::ffff:10.244.1.65\",\r\n", + " \"ips\": [],\r\n", + " \"protocol\": \"http\",\r\n", + " \"query\": {},\r\n", + " \"subdomains\": [],\r\n", + " \"xhr\": false,\r\n", + " \"os\": {\r\n", + " \"hostname\": \"logger-766f99b9b7-mqtql\"\r\n", + " },\r\n", + " \"connection\": {},\r\n", + " \"json\": {\r\n", + " \"data\": {\r\n", + " \"names\": [\r\n", + " \"proba\"\r\n", + " ],\r\n", + " \"ndarray\": [\r\n", + " [\r\n", + " 0.43782349911420193\r\n", + " ]\r\n", + " ]\r\n", + " },\r\n", + " \"meta\": {}\r\n", + " }\r\n", + "}\r\n", + "::ffff:10.244.1.65 - - [06/Nov/2020:09:35:09 +0000] \"POST / HTTP/1.1\" 200 1312 \"-\" \"Go-http-client/1.1\"\r\n" + ] + } + ], "source": [ "!kubectl logs $(kubectl get pods -l run=logger -n seldon -o jsonpath='{.items[0].metadata.name}') logger" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[' \"ce-modelid\": \"classifier\",', ' \"ce-modelid\": \"classifier\",']\n" + ] + } + ], "source": [ "modelids=!kubectl logs $(kubectl get pods -l run=logger -n seldon -o jsonpath='{.items[0].metadata.name}') logger | grep \"ce-modelid\"\n", "print(modelids)\n", - "assert(modelids[0].strip()==\"'ce-modelid': 'classifier',\")" + "assert(modelids[0].strip()==\"\\\"ce-modelid\\\": \\\"classifier\\\",\")" ] }, { @@ -180,18 +455,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"model-logs\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f model_logger.yaml -n seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment.apps \"logger\" deleted\r\n", + "service \"logger\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f message-dumper.yaml -n seldon" ] diff --git a/examples/models/sk_mnist/.gitignore b/examples/models/sk_mnist/.gitignore deleted file mode 100644 index f21e2a1a3c..0000000000 --- a/examples/models/sk_mnist/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -sk.pkl -MNIST_data/ diff --git a/examples/models/sk_mnist/Makefile b/examples/models/sk_mnist/Makefile deleted file mode 100644 index bde1c40298..0000000000 --- a/examples/models/sk_mnist/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -IMAGE_NAME=seldonio/sk-example-mnist -IMAGE_VERSION=0.3 - -train: - python train.py - -build_image: - s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:1.5.0-dev ${IMAGE_NAME}:${IMAGE_VERSION} - -push_image: - docker push $(IMAGE_NAME):$(IMAGE_VERSION) - -kind_load: - kind load -v 3 docker-image ${IMAGE_NAME}:${IMAGE_VERSION} - - -clean: - rm -f sk.pkl - rm -rf MNIST_data diff --git a/examples/models/sk_mnist/SkMnist.py b/examples/models/sk_mnist/SkMnist.py deleted file mode 100644 index 090d88500e..0000000000 --- a/examples/models/sk_mnist/SkMnist.py +++ /dev/null @@ -1,12 +0,0 @@ -from sklearn.externals import joblib - -class SkMnist(object): - def __init__(self): - self.class_names = ["class:{}".format(str(i)) for i in range(10)] - self.clf = joblib.load('sk.pkl') - - def predict(self,X,feature_names): - predictions = self.clf.predict_proba(X) - return predictions - - diff --git a/examples/models/sk_mnist/requirements.txt b/examples/models/sk_mnist/requirements.txt deleted file mode 100644 index 3393557ca5..0000000000 --- a/examples/models/sk_mnist/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -scipy>= 0.13.3 -scikit-learn>=0.18 diff --git a/examples/models/sk_mnist/train.py b/examples/models/sk_mnist/train.py deleted file mode 100644 index 066b277650..0000000000 --- a/examples/models/sk_mnist/train.py +++ /dev/null @@ -1,38 +0,0 @@ -from sklearn.ensemble import RandomForestClassifier -from sklearn import datasets, metrics -from sklearn.utils import shuffle -from sklearn.datasets import fetch_mldata -from sklearn.externals import joblib - -from tensorflow.examples.tutorials.mnist import input_data -mnist = input_data.read_data_sets("MNIST_data/") - -mnist_images = mnist.train.images -mnist_labels = mnist.train.labels - -# To apply a classifier on this data, we need to flatten the image, to -# turn the data in a (samples, feature) matrix: -n_samples = len(mnist_images) -data = mnist_images.reshape((n_samples, -1)) -targets = mnist_labels - -data,targets = shuffle(data,targets) -classifier = RandomForestClassifier(n_estimators=30) - -# We learn the digits on the first half of the digits -classifier.fit(data[:n_samples // 2], targets[:n_samples // 2]) - -# Now predict the value of the digit on the second half: -expected = targets[n_samples // 2:] -test_data = data[n_samples // 2:] - -print(classifier.score(test_data, expected)) - -predicted = classifier.predict(data[n_samples // 2:]) - -print("Classification report for classifier %s:\n%s\n" - % (classifier, metrics.classification_report(expected, predicted))) -print("Confusion matrix:\n%s" % metrics.confusion_matrix(expected, predicted)) - -joblib.dump(classifier, 'sk.pkl') - diff --git a/examples/models/sklearn_iris/Makefile b/examples/models/sklearn_iris/Makefile index 106a444c69..97278620b5 100644 --- a/examples/models/sklearn_iris/Makefile +++ b/examples/models/sklearn_iris/Makefile @@ -1,8 +1,8 @@ IMAGE_NAME=seldonio/sklearn-iris -IMAGE_VERSION=0.1 +IMAGE_VERSION=0.2 build_image: train - s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:1.5.0-dev ${IMAGE_NAME}:${IMAGE_VERSION} + s2i build -E environment . seldonio/seldon-core-s2i-python3:1.5.0-dev ${IMAGE_NAME}:${IMAGE_VERSION} push_image: docker push $(IMAGE_NAME):$(IMAGE_VERSION) diff --git a/examples/models/sklearn_iris/environment_rest b/examples/models/sklearn_iris/environment similarity index 100% rename from examples/models/sklearn_iris/environment_rest rename to examples/models/sklearn_iris/environment diff --git a/examples/models/sklearn_iris/sklearn_iris.ipynb b/examples/models/sklearn_iris/sklearn_iris.ipynb index 302cf9ef53..f128be2466 100644 --- a/examples/models/sklearn_iris/sklearn_iris.ipynb +++ b/examples/models/sklearn_iris/sklearn_iris.ipynb @@ -19,138 +19,9 @@ "pip install seldon-core\n", "```\n", "\n", - "## Train locally\n", " " ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile train_iris.py\n", - "import joblib\n", - "from sklearn.pipeline import Pipeline\n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn import datasets\n", - "\n", - "\n", - "OUTPUT_FILE = \"IrisClassifier.sav\"\n", - "\n", - "\n", - "def main():\n", - " clf = LogisticRegression(solver=\"liblinear\", multi_class=\"ovr\")\n", - " p = Pipeline([(\"clf\", clf)])\n", - " print(\"Training model...\")\n", - " p.fit(X, y)\n", - " print(\"Model trained!\")\n", - "\n", - " print(f\"Saving model in {OUTPUT_FILE}\")\n", - " joblib.dump(p, OUTPUT_FILE)\n", - " print(\"Model saved!\")\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " print(\"Loading iris data set...\")\n", - " iris = datasets.load_iris()\n", - " X, y = iris.data, iris.target\n", - " print(\"Dataset loaded!\")\n", - "\n", - " main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python train_iris.py" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Wrap model with Python Wrapper Class" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile IrisClassifier.py\n", - "import joblib\n", - "\n", - "class IrisClassifier(object):\n", - "\n", - " def __init__(self):\n", - " self.model = joblib.load('IrisClassifier.sav')\n", - "\n", - " def predict(self,X,features_names):\n", - " return self.model.predict_proba(X)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Wrap model using s2i" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## REST test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:1.5.0-dev seldonio/sklearn-iris:0.1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!docker run --name \"iris_predictor\" -d --rm -p 5000:5000 seldonio/sklearn-iris:0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Send some random features that conform to the contract" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!curl -s http://localhost:5000/predict -H \"Content-Type: application/json\" -d '{\"data\":{\"ndarray\":[[5.964,4.006,2.081,1.031]]}}'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!docker rm iris_predictor --force" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -167,18 +38,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] @@ -192,9 +79,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting sklearn_iris_deployment.yaml\n" + ] + } + ], "source": [ "%%writefile sklearn_iris_deployment.yaml\n", "apiVersion: machinelearning.seldon.io/v1alpha2\n", @@ -207,7 +102,7 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/sklearn-iris:0.1\n", + " - image: seldonio/sklearn-iris:0.2\n", " imagePullPolicy: IfNotPresent\n", " name: sklearn-iris-classifier\n", " graph:\n", @@ -222,18 +117,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/seldon-deployment-example created\r\n" + ] + } + ], "source": [ "!kubectl create -f sklearn_iris_deployment.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"seldon-92a927e5e90d7602e08ba9b9304f70e8\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"seldon-92a927e5e90d7602e08ba9b9304f70e8\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=seldon-deployment-example \\\n", " -o jsonpath='{.items[0].metadata.name}')" @@ -241,9 +153,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], "source": [ "for i in range(60):\n", " state=!kubectl get sdep seldon-deployment-example -o jsonpath='{.status.state}'\n", @@ -257,7 +177,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -266,18 +186,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.9548873249364059,0.04505474761562512,5.7927447968953825e-05]]},\"meta\":{}}']" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "res" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['{\"data\":{\"names\":[\"t:0\",\"t:1\",\"t:2\"],\"ndarray\":[[0.9548873249364059,0.04505474761562512,5.7927447968953825e-05]]},\"meta\":{}}']\n" + ] + } + ], "source": [ "print(res)\n", "import json\n", @@ -287,9 +226,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"seldon-deployment-example\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f sklearn_iris_deployment.yaml" ] diff --git a/examples/models/sklearn_iris/sklearn_iris_deployment.yaml b/examples/models/sklearn_iris/sklearn_iris_deployment.yaml index a794ed6648..f4e942a791 100644 --- a/examples/models/sklearn_iris/sklearn_iris_deployment.yaml +++ b/examples/models/sklearn_iris/sklearn_iris_deployment.yaml @@ -8,7 +8,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/sklearn-iris:0.1 + - image: seldonio/sklearn-iris:0.2 imagePullPolicy: IfNotPresent name: sklearn-iris-classifier graph: @@ -18,4 +18,4 @@ spec: name: sklearn-iris-classifier type: MODEL name: sklearn-iris-predictor - replicas: 1 \ No newline at end of file + replicas: 1 diff --git a/examples/models/sklearn_iris_jsondata/Makefile b/examples/models/sklearn_iris_jsondata/Makefile index 2ebb3abe23..c13d481de6 100644 --- a/examples/models/sklearn_iris_jsondata/Makefile +++ b/examples/models/sklearn_iris_jsondata/Makefile @@ -1,8 +1,8 @@ IMAGE_NAME=seldonio/sklearn-iris-jsondata -IMAGE_VERSION=0.1 +IMAGE_VERSION=0.2 build_image: train - s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:1.5.0-dev ${IMAGE_NAME}:${IMAGE_VERSION} + s2i build -E environment . seldonio/seldon-core-s2i-python3:1.5.0-dev ${IMAGE_NAME}:${IMAGE_VERSION} push_image: docker push $(IMAGE_NAME):$(IMAGE_VERSION) diff --git a/examples/models/sklearn_iris_jsondata/environment_rest b/examples/models/sklearn_iris_jsondata/environment similarity index 80% rename from examples/models/sklearn_iris_jsondata/environment_rest rename to examples/models/sklearn_iris_jsondata/environment index 16c663c9a4..76871a2f7d 100644 --- a/examples/models/sklearn_iris_jsondata/environment_rest +++ b/examples/models/sklearn_iris_jsondata/environment @@ -1,4 +1,3 @@ MODEL_NAME=IrisClassifier -API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata.ipynb b/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata.ipynb index 8295c07611..cdb46f21c3 100644 --- a/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata.ipynb +++ b/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata.ipynb @@ -19,155 +19,9 @@ "pip install seldon-core\n", "```\n", "\n", - "## Train locally\n", " " ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile train_iris.py\n", - "import joblib\n", - "from sklearn.pipeline import Pipeline\n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn import datasets\n", - "\n", - "\n", - "OUTPUT_FILE = \"IrisClassifier.sav\"\n", - "\n", - "\n", - "def main():\n", - " clf = LogisticRegression(solver=\"liblinear\", multi_class=\"ovr\")\n", - " p = Pipeline([(\"clf\", clf)])\n", - " print(\"Training model...\")\n", - " p.fit(X, y)\n", - " print(\"Model trained!\")\n", - "\n", - " print(f\"Saving model in {OUTPUT_FILE}\")\n", - " joblib.dump(p, OUTPUT_FILE)\n", - " print(\"Model saved!\")\n", - "\n", - "\n", - "if __name__ == \"__main__\":\n", - " print(\"Loading iris data set...\")\n", - " iris = datasets.load_iris()\n", - " X, y = iris.data, iris.target\n", - " print(\"Dataset loaded!\")\n", - "\n", - " main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!python train_iris.py" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Wrap your model with a Python wrapper" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile IrisClassifier.py\n", - "import joblib\n", - "import sys\n", - "\n", - "\n", - "def eprint(*args, **kwargs):\n", - " print(*args, file=sys.stderr, **kwargs)\n", - "\n", - "\n", - "class IrisClassifier(object):\n", - " def __init__(self):\n", - " self.model = joblib.load(\"IrisClassifier.sav\")\n", - "\n", - " def predict(self, X, features_names):\n", - " eprint(\"--------------------\")\n", - " eprint(\"Input dict\")\n", - " eprint(X)\n", - " eprint(\"--------------------\")\n", - " ndarray = X[\"some_data\"][\"some_ndarray\"]\n", - " return self.model.predict_proba(ndarray)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## REST test" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Wrap your Python model using s2i" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!s2i build -E environment_rest . seldonio/seldon-core-s2i-python3:1.5.0-dev seldonio/sklearn-iris-jsondata:0.1" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Serve model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!docker run --name \"iris_predictor\" -d --rm -p 5000:5000 seldonio/sklearn-iris-jsondata:0.1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!curl -sg http://localhost:5000/predict --data-urlencode 'json={\"jsonData\": {\"some_data\": {\"names\": [\"sepal_length\",\"sepal_width\",\"petal_length\",\"petal_width\"],\"some_ndarray\": [[7.233,4.652,7.39,0.324]]}}}' " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Stop serving model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!docker rm iris_predictor --force" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -184,18 +38,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] @@ -209,9 +79,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting sklearn_iris_jsondata_deployment.yaml\n" + ] + } + ], "source": [ "%%writefile sklearn_iris_jsondata_deployment.yaml\n", "apiVersion: machinelearning.seldon.io/v1alpha2\n", @@ -224,7 +102,7 @@ " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/sklearn-iris-jsondata:0.1\n", + " - image: seldonio/sklearn-iris-jsondata:0.2\n", " imagePullPolicy: IfNotPresent\n", " name: sklearn-iris-classifier\n", " graph:\n", @@ -239,18 +117,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/seldon-deployment-example created\r\n" + ] + } + ], "source": [ "!kubectl create -f sklearn_iris_jsondata_deployment.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"seldon-92a927e5e90d7602e08ba9b9304f70e8\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"seldon-92a927e5e90d7602e08ba9b9304f70e8\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=seldon-deployment-example -o jsonpath='{.items[0].metadata.name}')" ] @@ -271,9 +166,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['t:0', 't:1', 't:2'], 'tensor': {'shape': [1, 3], 'values': [0.0011707012599976396, 0.7876686831680301, 0.21116061557197244]}}, 'meta': {}}\n" + ] + } + ], "source": [ "import json\n", "res=!curl -s -H \"Content-Type: application/json\" \\\n", @@ -293,9 +196,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['t:0', 't:1', 't:2'], 'tensor': {'shape': [1, 3], 'values': [0.0011707012599976396, 0.7876686831680301, 0.21116061557197244]}}, 'meta': {}}\n" + ] + } + ], "source": [ "res=!curl -s -H 'Content-Type:multipart/form-data' \\\n", " -F jsonData='{\"some_data\": {\"names\": [\"sepal_length\",\"sepal_width\",\"petal_length\",\"petal_width\"],\"some_ndarray\": [[7.233,4.652,7.39,0.324]]}}' \\\n", @@ -307,9 +218,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"seldon-deployment-example\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f sklearn_iris_jsondata_deployment.yaml" ] @@ -338,7 +257,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.6.8" }, "varInspector": { "cols": { diff --git a/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata_deployment.yaml b/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata_deployment.yaml index 9c4bad8b5a..84a4951203 100644 --- a/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata_deployment.yaml +++ b/examples/models/sklearn_iris_jsondata/sklearn_iris_jsondata_deployment.yaml @@ -8,7 +8,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/sklearn-iris-jsondata:0.1 + - image: seldonio/sklearn-iris-jsondata:0.2 imagePullPolicy: IfNotPresent name: sklearn-iris-classifier graph: diff --git a/executor/Makefile b/executor/Makefile index cca73eb4cb..7f1a356dfd 100644 --- a/executor/Makefile +++ b/executor/Makefile @@ -35,7 +35,7 @@ kafka-proxy: copy_operator fmt vet go build -o kafka-proxy cmd/proxy/main.go -.PHONE: copy_operator +.PHONY: copy_operator copy_operator: rm _operator -rf cp -r ../operator _operator diff --git a/executor/api/client/client.go b/executor/api/client/client.go index d868a38595..d16035761e 100644 --- a/executor/api/client/client.go +++ b/executor/api/client/client.go @@ -35,6 +35,7 @@ type SeldonApiClient interface { Unmarshall(msg []byte, contentType string) (payload.SeldonPayload, error) Marshall(out io.Writer, msg payload.SeldonPayload) error CreateErrorPayload(err error) payload.SeldonPayload + IsGrpc() bool } type SeldonApiError struct { diff --git a/executor/api/grpc/kfserving/client.go b/executor/api/grpc/kfserving/client.go index 11ef32bab6..8a0960cc76 100644 --- a/executor/api/grpc/kfserving/client.go +++ b/executor/api/grpc/kfserving/client.go @@ -25,6 +25,10 @@ type KFServingGrpcClient struct { annotations map[string]string } +func (s *KFServingGrpcClient) IsGrpc() bool { + return true +} + func (s *KFServingGrpcClient) ModelMetadata(ctx context.Context, modelName string, host string, port int32, msg payload.SeldonPayload, meta map[string][]string) (payload.ModelMetadata, error) { panic("implement me") } diff --git a/executor/api/grpc/seldon/client.go b/executor/api/grpc/seldon/client.go index c0c1b06aac..363fe46341 100644 --- a/executor/api/grpc/seldon/client.go +++ b/executor/api/grpc/seldon/client.go @@ -29,6 +29,10 @@ type SeldonMessageGrpcClient struct { annotations map[string]string } +func (s *SeldonMessageGrpcClient) IsGrpc() bool { + return true +} + func NewSeldonGrpcClient(spec *v1.PredictorSpec, deploymentName string, annotations map[string]string) client.SeldonApiClient { opts := []grpc.CallOption{ grpc.MaxCallSendMsgSize(math.MaxInt32), diff --git a/executor/api/grpc/tensorflow/client.go b/executor/api/grpc/tensorflow/client.go index 66e1920e58..e1b3e5558f 100644 --- a/executor/api/grpc/tensorflow/client.go +++ b/executor/api/grpc/tensorflow/client.go @@ -28,6 +28,10 @@ type TensorflowGrpcClient struct { annotations map[string]string } +func (s *TensorflowGrpcClient) IsGrpc() bool { + return true +} + func NewTensorflowGrpcClient(predictor *v1.PredictorSpec, deploymentName string, annotations map[string]string) client.SeldonApiClient { opts := []grpc.CallOption{ grpc.MaxCallSendMsgSize(math.MaxInt32), diff --git a/executor/api/grpc/tensorflow/server_test.go b/executor/api/grpc/tensorflow/server_test.go index 6f9921897b..ac07e5804a 100644 --- a/executor/api/grpc/tensorflow/server_test.go +++ b/executor/api/grpc/tensorflow/server_test.go @@ -28,6 +28,10 @@ type TestTensorflowClient struct { t *testing.T } +func (s TestTensorflowClient) IsGrpc() bool { + return true +} + func NewTestTensorflowClient(t *testing.T) client.SeldonApiClient { client := TestTensorflowClient{ t: t, diff --git a/executor/api/kafka/client.go b/executor/api/kafka/client.go index 396f68808d..4f155ad0d5 100644 --- a/executor/api/kafka/client.go +++ b/executor/api/kafka/client.go @@ -25,6 +25,10 @@ type KafkaClient struct { topicHandlers map[string]*KafkaRPC } +func (kc *KafkaClient) IsGrpc() bool { + return false +} + func NewKafkaClient(hostname, deploymentName, namespace, protocol, transport string, predictor *v1.PredictorSpec, broker string, log logr.Logger) client.SeldonApiClient { skc := &KafkaClient{ Hostname: hostname, diff --git a/executor/api/rest/client.go b/executor/api/rest/client.go index 25c400d815..7965413088 100644 --- a/executor/api/rest/client.go +++ b/executor/api/rest/client.go @@ -44,6 +44,10 @@ type JSONRestClient struct { metrics *metric.ClientMetrics } +func (smc *JSONRestClient) IsGrpc() bool { + return false +} + func (smc *JSONRestClient) CreateErrorPayload(err error) payload.SeldonPayload { respFailed := proto.SeldonMessage{Status: &proto.Status{Code: http.StatusInternalServerError, Info: err.Error()}} m := jsonpb.Marshaler{} diff --git a/executor/api/rest/server_test.go b/executor/api/rest/server_test.go index 9eea3463d7..2920b2fe6d 100644 --- a/executor/api/rest/server_test.go +++ b/executor/api/rest/server_test.go @@ -204,6 +204,7 @@ func TestRequestPuuidHeaderIsSet(t *testing.T) { ServiceHost: urlParts[0], ServicePort: int32(port), Type: v1.REST, + HttpPort: int32(port), }, }, } @@ -284,6 +285,7 @@ func TestModelWithServer(t *testing.T) { ServiceHost: urlParts[0], ServicePort: int32(port), Type: v1.REST, + HttpPort: int32(port), }, }, } @@ -588,6 +590,7 @@ func TestPredictErrorWithServer(t *testing.T) { ServiceHost: urlParts[0], ServicePort: int32(port), Type: v1.REST, + HttpPort: int32(port), }, }, } diff --git a/executor/api/test/seldonmessage_test_client.go b/executor/api/test/seldonmessage_test_client.go index f6a670e479..f5c3ece3b2 100644 --- a/executor/api/test/seldonmessage_test_client.go +++ b/executor/api/test/seldonmessage_test_client.go @@ -44,6 +44,10 @@ const ( }` ) +func (s SeldonMessageTestClient) IsGrpc() bool { + return true +} + func (s SeldonMessageTestClient) Status(ctx context.Context, modelName string, host string, port int32, msg payload.SeldonPayload, meta map[string][]string) (payload.SeldonPayload, error) { return &payload.BytesPayload{Msg: []byte(TestClientStatusResponse)}, nil } diff --git a/executor/cmd/executor/main.go b/executor/cmd/executor/main.go index 462578a8e0..4ede05a256 100644 --- a/executor/cmd/executor/main.go +++ b/executor/cmd/executor/main.go @@ -57,8 +57,8 @@ var ( sdepName = flag.String("sdep", "", "Seldon deployment name") namespace = flag.String("namespace", "", "Namespace") predictorName = flag.String("predictor", "", "Name of the predictor inside the SeldonDeployment") - httpPort = flag.Int("http_port", 8080, "Executor port") - grpcPort = flag.Int("grpc_port", 8000, "Executor port") + httpPort = flag.Int("http_port", 8080, "Executor http port") + grpcPort = flag.Int("grpc_port", 5000, "Executor grpc port") wait = flag.Duration("graceful_timeout", time.Second*15, "Graceful shutdown secs") protocol = flag.String("protocol", "seldon", "The payload protocol") transport = flag.String("transport", "rest", "The network transport mechanism rest, grpc") @@ -309,31 +309,28 @@ func main() { }() } - if *transport == "rest" { - clientRest, err := rest.NewJSONRestClient(*protocol, *sdepName, predictor, annotations) - if err != nil { - log.Fatalf("Failed to create http client: %v", err) - } - logger.Info("Running http server ", "port", *httpPort) - runHttpServer(createListener(*httpPort, logger), logger, predictor, clientRest, *httpPort, false, serverUrl, *namespace, *protocol, *sdepName, *prometheusPath) - } else { - logger.Info("Running http probes only server ", "port", *httpPort) - go runHttpServer(createListener(*httpPort, logger), logger, predictor, nil, *httpPort, true, serverUrl, *namespace, *protocol, *sdepName, *prometheusPath) - logger.Info("Running grpc server ", "port", *grpcPort) - var clientGrpc seldonclient.SeldonApiClient - - switch *protocol { - case api.ProtocolSeldon: - clientGrpc = seldon.NewSeldonGrpcClient(predictor, *sdepName, annotations) - case api.ProtocolTensorflow: - clientGrpc = tensorflow.NewTensorflowGrpcClient(predictor, *sdepName, annotations) - case api.ProtocolKFServing: - clientGrpc = kfserving.NewKFServingGrpcClient(predictor, *sdepName, annotations) - default: - log.Fatalf("Failed to create grpc client. Unknown protocol %s: %v", *protocol, err) - } - runGrpcServer(createListener(*grpcPort, logger), logger, predictor, clientGrpc, serverUrl, *namespace, *protocol, *sdepName, annotations) + clientRest, err := rest.NewJSONRestClient(*protocol, *sdepName, predictor, annotations) + if err != nil { + log.Fatalf("Failed to create http client: %v", err) + } + + var clientGrpc seldonclient.SeldonApiClient + switch *protocol { + case api.ProtocolSeldon: + clientGrpc = seldon.NewSeldonGrpcClient(predictor, *sdepName, annotations) + case api.ProtocolTensorflow: + clientGrpc = tensorflow.NewTensorflowGrpcClient(predictor, *sdepName, annotations) + case api.ProtocolKFServing: + clientGrpc = kfserving.NewKFServingGrpcClient(predictor, *sdepName, annotations) + default: + log.Fatalf("Failed to create grpc client. Unknown protocol %s: %v", *protocol, err) } + + logger.Info("Running http server ", "port", *httpPort) + go runHttpServer(createListener(*httpPort, logger), logger, predictor, clientRest, *httpPort, false, serverUrl, *namespace, *protocol, *sdepName, *prometheusPath) + + logger.Info("Running grpc server ", "port", *grpcPort) + runGrpcServer(createListener(*grpcPort, logger), logger, predictor, clientGrpc, serverUrl, *namespace, *protocol, *sdepName, annotations) } func createListener(port int, logger logr.Logger) net.Listener { diff --git a/executor/go.mod b/executor/go.mod index 0ad8c7cacb..030d413376 100644 --- a/executor/go.mod +++ b/executor/go.mod @@ -7,6 +7,7 @@ require ( github.com/confluentinc/confluent-kafka-go v1.4.2 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v0.1.0 + github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.4.2 github.com/google/uuid v1.1.2 github.com/gorilla/mux v1.8.0 diff --git a/executor/predictor/predictor_process.go b/executor/predictor/predictor_process.go index 0ef92acf6a..a0d7ab4cb8 100644 --- a/executor/predictor/predictor_process.go +++ b/executor/predictor/predictor_process.go @@ -55,6 +55,14 @@ func hasMethod(method v1.PredictiveUnitMethod, methods *[]v1.PredictiveUnitMetho return false } +func (p *PredictorProcess) getPort(node *v1.PredictiveUnit) int32 { + if p.Client.IsGrpc() { + return node.Endpoint.GrpcPort + } else { + return node.Endpoint.HttpPort + } +} + func (p *PredictorProcess) transformInput(node *v1.PredictiveUnit, msg payload.SeldonPayload) (payload.SeldonPayload, error) { callModel := false callTransformInput := false @@ -74,13 +82,13 @@ func (p *PredictorProcess) transformInput(node *v1.PredictiveUnit, msg payload.S if err != nil { return nil, err } - return p.Client.Predict(p.Ctx, node.Name, node.Endpoint.ServiceHost, node.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.Predict(p.Ctx, node.Name, node.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } else if callTransformInput { msg, err := p.Client.Chain(p.Ctx, node.Name, msg) if err != nil { return nil, err } - return p.Client.TransformInput(p.Ctx, node.Name, node.Endpoint.ServiceHost, node.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.TransformInput(p.Ctx, node.Name, node.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } else { return msg, nil } @@ -104,7 +112,7 @@ func (p *PredictorProcess) transformOutput(node *v1.PredictiveUnit, msg payload. if err != nil { return nil, err } - return p.Client.TransformOutput(p.Ctx, node.Name, node.Endpoint.ServiceHost, node.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.TransformOutput(p.Ctx, node.Name, node.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } else { return msg, nil } @@ -124,7 +132,7 @@ func (p *PredictorProcess) feedback(node *v1.PredictiveUnit, msg payload.SeldonP } if callClient { - return p.Client.Feedback(p.Ctx, node.Name, node.Endpoint.ServiceHost, node.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.Feedback(p.Ctx, node.Name, node.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } else { return msg, nil } @@ -143,7 +151,7 @@ func (p *PredictorProcess) route(node *v1.PredictiveUnit, msg payload.SeldonPayl callClient = true } if callClient { - return p.Client.Route(p.Ctx, node.Name, node.Endpoint.ServiceHost, node.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.Route(p.Ctx, node.Name, node.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } else if node.Implementation != nil && *node.Implementation == v1.RANDOM_ABTEST { return p.abTestRouter(node) } else { @@ -164,7 +172,7 @@ func (p *PredictorProcess) aggregate(node *v1.PredictiveUnit, msg []payload.Seld } if callClient { - return p.Client.Combine(p.Ctx, node.Name, node.Endpoint.ServiceHost, node.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.Combine(p.Ctx, node.Name, node.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } else { return msg[0], nil } @@ -321,7 +329,7 @@ func (p *PredictorProcess) Status(node *v1.PredictiveUnit, modelName string, msg if nodeModel := v1.GetPredictiveUnit(node, modelName); nodeModel == nil { return nil, fmt.Errorf("Failed to find model %s", modelName) } else { - return p.Client.Status(p.Ctx, modelName, nodeModel.Endpoint.ServiceHost, nodeModel.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.Status(p.Ctx, modelName, nodeModel.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } } @@ -329,7 +337,7 @@ func (p *PredictorProcess) Metadata(node *v1.PredictiveUnit, modelName string, m if nodeModel := v1.GetPredictiveUnit(node, modelName); nodeModel == nil { return nil, fmt.Errorf("Failed to find model %s", modelName) } else { - return p.Client.Metadata(p.Ctx, modelName, nodeModel.Endpoint.ServiceHost, nodeModel.Endpoint.ServicePort, msg, p.Meta.Meta) + return p.Client.Metadata(p.Ctx, modelName, nodeModel.Endpoint.ServiceHost, p.getPort(node), msg, p.Meta.Meta) } } @@ -373,7 +381,7 @@ func (p *PredictorProcess) Feedback(node *v1.PredictiveUnit, msg payload.SeldonP } func (p *PredictorProcess) ModelMetadataMap(node *v1.PredictiveUnit) (map[string]payload.ModelMetadata, error) { - resPayload, err := p.Client.ModelMetadata(p.Ctx, node.Name, node.Endpoint.ServiceHost, node.Endpoint.ServicePort, nil, p.Meta.Meta) + resPayload, err := p.Client.ModelMetadata(p.Ctx, node.Name, node.Endpoint.ServiceHost, p.getPort(node), nil, p.Meta.Meta) if err != nil { return nil, err } diff --git a/executor/samples/local/graph/Makefile b/executor/samples/local/graph/Makefile index b06a4b030f..0838cdfb04 100644 --- a/executor/samples/local/graph/Makefile +++ b/executor/samples/local/graph/Makefile @@ -2,23 +2,26 @@ BASE=../../.. ## REST -run_rest_executor: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_rest.yaml --http_port 8000 +run_executor: + ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model.yaml --http_port 8000 --grpc_port 5000 run_model: - export PREDICTIVE_UNIT_SERVICE_PORT=9001 && export PREDICTIVE_UNIT_METRICS_SERVICE_PORT=6001 && seldon-core-microservice --service-type MODEL Model REST + export PREDICTIVE_UNIT_HTTP_SERVICE_PORT=9001 && export PREDICTIVE_UNIT_GRPC_SERVICE_PORT=9501 && export PREDICTIVE_UNIT_METRICS_SERVICE_PORT=6001 && seldon-core-microservice --service-type MODEL Model GRPC run_transformer: - export PREDICTIVE_UNIT_SERVICE_PORT=9000 && seldon-core-microservice --service-type TRANSFORMER Transformer REST + export PREDICTIVE_UNIT_HTTP_SERVICE_PORT=9000 && export PREDICTIVE_UNIT_GRPC_SERVICE_PORT=9500 && seldon-core-microservice --service-type TRANSFORMER Transformer GRPC curl_rest: - curl -v localhost:8000/api/v0.1/predictions -H "Accept: application/json" -H "Content-Type: application/json" -d '{"jsonData":"{\"data\":{\"ndarray\":[[1.0,2.0]]}}"}' + curl -v localhost:8000/api/v0.1/predictions -H "Accept: application/json" -H "Content-Type: application/json" -d '{"data":{"ndarray":[[1.0,2.0]]}}' curl_rest_multipart: curl -v localhost:8000/api/v0.1/predictions -H "Accept: application/json" -F jsonData=@input.json +grpc_test: + cd ${BASE}/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' -plaintext -proto ./prediction.proto 0.0.0.0:5000 seldon.protos.Seldon/Predict + diff --git a/executor/samples/local/graph/Transformer.py b/executor/samples/local/graph/Transformer.py index b31b9455e1..a6e09207d3 100644 --- a/executor/samples/local/graph/Transformer.py +++ b/executor/samples/local/graph/Transformer.py @@ -2,6 +2,6 @@ class Transformer(object): - def transform_input_raw(self, X): + def transform_input(self, X, meta): print(X) - return json.loads(X["jsonData"]) \ No newline at end of file + return X+1 diff --git a/executor/samples/local/graph/model_rest.yaml b/executor/samples/local/graph/model.yaml similarity index 84% rename from executor/samples/local/graph/model_rest.yaml rename to executor/samples/local/graph/model.yaml index f1706e54a4..5eeb127761 100644 --- a/executor/samples/local/graph/model_rest.yaml +++ b/executor/samples/local/graph/model.yaml @@ -21,6 +21,8 @@ spec: type: REST service_host: 0.0.0.0 service_port: 9000 + http_port: 9000 + grpc_port: 9500 name: transformer type: TRANSFORMER children: @@ -28,7 +30,9 @@ spec: type: REST service_host: 0.0.0.0 service_port: 9001 - name: classifier + name: classifier + http_port: 9001 + grpc_port: 9501 type: MODEL labels: version: v1 diff --git a/executor/samples/local/kfserving/Makefile b/executor/samples/local/kfserving/Makefile index f5e16eadfb..23611771ef 100644 --- a/executor/samples/local/kfserving/Makefile +++ b/executor/samples/local/kfserving/Makefile @@ -8,12 +8,12 @@ triton-inference-server/docs/examples/model_repository/simple: triton-inference- ## REST -run_rest_executor: - ${BASE}/executor --sdep triton --namespace default --predictor simple --file ./model_rest.yaml --http_port 8000 --protocol kfserving +run_executor: + ${BASE}/executor --sdep triton --namespace default --predictor simple --file ./model.yaml --http_port 8000 --grpc_port 5000 --protocol kfserving -run_triton_simple_rest: - docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -v ${PWD}/triton-inference-server/docs/examples/model_repository:/models nvcr.io/nvidia/tritonserver:20.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --http-port=9000 +run_triton_simple: + docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -p5001:5001 -v ${PWD}/triton-inference-server/docs/examples/model_repository:/models nvcr.io/nvidia/tritonserver:20.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --http-port=9000 --grpc-port=5001 curl_rest_triton: curl -v -d '{"inputs":[{"name":"INPUT0","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],"datatype":"INT32","shape":[1,16]}]}' -X POST http://0.0.0.0:9000/v2/models/simple/infer -H "Content-Type: application/json" @@ -29,26 +29,17 @@ curl_metadata: curl http://localhost:8000/v2/models/simple -## GRPC - -run_grpc_executor: - ${BASE}/executor --sdep triton --namespace default --predictor simple --file ./model_rest.yaml --grpc_port 8000 --protocol kfserving --transport grpc - -run_triton_simple_grpc: - docker run --rm --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 -p9000:9000 -p8001:8001 -p8002:8002 -v ${PWD}/triton-inference-server/docs/examples/model_repository:/models nvcr.io/nvidia/tritonserver:20.08-py3 /opt/tritonserver/bin/tritonserver --model-repository=/models --grpc-port=9000 - - grpc_test: - cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"model_name":"simple","inputs":[{"name":"INPUT0","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]}]}' -plaintext -proto ./grpc_service.proto 0.0.0.0:8000 inference.GRPCInferenceService/ModelInfer + cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"model_name":"simple","inputs":[{"name":"INPUT0","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]},{"name":"INPUT1","contents":{"int_contents":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},"datatype":"INT32","shape":[1,16]}]}' -plaintext -proto ./grpc_service.proto 0.0.0.0:5000 inference.GRPCInferenceService/ModelInfer grpc_status: - cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"name":"simple"}' -plaintext -proto ./grpc_service.proto 0.0.0.0:8000 inference.GRPCInferenceService/ModelReady + cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"name":"simple"}' -plaintext -proto ./grpc_service.proto 0.0.0.0:5000 inference.GRPCInferenceService/ModelReady grpc_status_triton: - cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"name":"simple"}' -plaintext -proto ./grpc_service.proto 0.0.0.0:8000 inference.GRPCInferenceService/ModelReady + cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"name":"simple"}' -plaintext -proto ./grpc_service.proto 0.0.0.0:5001 inference.GRPCInferenceService/ModelReady grpc_metadata: - cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"name":"simple"}' -plaintext -proto ./grpc_service.proto 0.0.0.0:8000 inference.GRPCInferenceService/ModelMetadata + cd ${BASE}/api/grpc/kfserving/inference && grpcurl -d '{"name":"simple"}' -plaintext -proto ./grpc_service.proto 0.0.0.0:5000 inference.GRPCInferenceService/ModelMetadata diff --git a/executor/samples/local/kfserving/model_rest.yaml b/executor/samples/local/kfserving/model.yaml similarity index 87% rename from executor/samples/local/kfserving/model_rest.yaml rename to executor/samples/local/kfserving/model.yaml index 34356baed5..8c60d45699 100644 --- a/executor/samples/local/kfserving/model_rest.yaml +++ b/executor/samples/local/kfserving/model.yaml @@ -8,9 +8,9 @@ spec: - graph: children: [] endpoint: - type: REST service_host: 0.0.0.0 - service_port: 9000 + http_port: 9000 + grpc_port: 5001 implementation: TRITON_SERVER modelUri: gs://seldon-models/trtis/simple-model name: simple diff --git a/executor/samples/local/logger/Makefile b/executor/samples/local/logger/Makefile index cf36309844..ef9706111b 100644 --- a/executor/samples/local/logger/Makefile +++ b/executor/samples/local/logger/Makefile @@ -2,12 +2,12 @@ BASE=../../.. ## REST -run_rest_executor: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_rest.yaml --http_port 8000 +run_executor: + ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model.yaml --http_port 8000 --grpc_port 9500 -run_dummy_rest_model: - cd ${BASE}/../examples/models/mean_classifier && make run_rest_local +run_dummy_model: + cd ${BASE}/../examples/models/mean_classifier && make run_local run_dummy_logsink: docker run -it -p 2222:80 --rm -t mendhak/http-https-echo @@ -18,17 +18,7 @@ curl_rest: curl_rest_big: curl -v localhost:8000/api/v0.1/predictions -H "Accept: application/json" -H "Content-Type: application/json" -d '{"data":{"names":["Age","Workclass","Education","Marital Status","Occupation","Relationship","Race","Sex","Capital Gain","Capital Loss","Hours per week","Country"],"ndarray":[[53,4,0,2,8,4,2,0,0,0,60,9]]}}' - - -## GRPC - -run_grpc_executor: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_grpc.yaml --http_port 8000 --grpc_port 5000 --transport grpc - -run_dummy_grpc_model: - cd ${BASE}/../examples/models/mean_classifier && make run_grpc_local - grpc_test: - cd ${BASE}/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' -plaintext -proto ./prediction.proto 0.0.0.0:5000 seldon.protos.Seldon/Predict + cd ${BASE}/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' -plaintext -proto ./prediction.proto 0.0.0.0:9500 seldon.protos.Seldon/Predict diff --git a/executor/samples/local/logger/model_rest.yaml b/executor/samples/local/logger/model.yaml similarity index 92% rename from executor/samples/local/logger/model_rest.yaml rename to executor/samples/local/logger/model.yaml index 3c61e616a6..c61bf09cab 100644 --- a/executor/samples/local/logger/model_rest.yaml +++ b/executor/samples/local/logger/model.yaml @@ -17,9 +17,9 @@ spec: graph: children: [] endpoint: - type: REST service_host: 0.0.0.0 - service_port: 9000 + http_port: 9001 + grpc_port: 5001 name: classifier type: MODEL logger: diff --git a/executor/samples/local/logger/model_grpc.yaml b/executor/samples/local/logger/model_grpc.yaml deleted file mode 100644 index bd2b11e2c3..0000000000 --- a/executor/samples/local/logger/model_grpc.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1alpha2 -kind: SeldonDeployment -metadata: - labels: - app: seldon - name: seldon-model -spec: - annotations: - seldon.io/executor: "true" - name: test-deployment - predictors: - - componentSpecs: - - spec: - containers: - - image: seldonio/mock_classifier_rest:1.3 - name: classifier - graph: - children: [] - endpoint: - type: GRPC - service_host: 0.0.0.0 - service_port: 9000 - name: classifier - type: MODEL - logger: - mode: request - url: http://localhost:2222 - labels: - version: v1 - name: example - replicas: 1 diff --git a/executor/samples/local/metrics/Makefile b/executor/samples/local/metrics/Makefile index 27397e13ba..b4ec2f77f4 100644 --- a/executor/samples/local/metrics/Makefile +++ b/executor/samples/local/metrics/Makefile @@ -2,12 +2,12 @@ BASE=../../.. ## REST -run_rest_executor: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_rest.yaml --http_port 8000 +run_executor: + ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model.yaml --http_port 8000 --grpc_port 9500 -run_dummy_rest_model: - cd ${BASE}/../examples/models/mean_classifier && make run_rest_local +run_dummy_model: + cd ${BASE}/../examples/models/mean_classifier && make run_local curl_rest: curl -v localhost:8000/api/v0.1/predictions -d '{"data":{"ndarray":[[1.0,2.0]]}}' -H "Accept: application/json" -H "Content-Type: application/json" @@ -15,15 +15,7 @@ curl_rest: curl_metrics: curl localhost:8000/metrics -## GRPC - -run_grpc_executor: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_grpc.yaml --http_port 8000 --grpc_port 5000 --transport grpc - -run_dummy_grpc_model: - cd ${BASE}/../examples/models/mean_classifier && make run_grpc_local - grpc_test: - cd ${BASE}/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' -plaintext -proto ./prediction.proto 0.0.0.0:8000 seldon.protos.Seldon/Predict + cd ${BASE}/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' -plaintext -proto ./prediction.proto 0.0.0.0:9500 seldon.protos.Seldon/Predict diff --git a/executor/samples/local/tracing/model_rest.yaml b/executor/samples/local/metrics/model.yaml similarity index 91% rename from executor/samples/local/tracing/model_rest.yaml rename to executor/samples/local/metrics/model.yaml index 0bece0ef37..d09c540d85 100644 --- a/executor/samples/local/tracing/model_rest.yaml +++ b/executor/samples/local/metrics/model.yaml @@ -19,7 +19,8 @@ spec: endpoint: type: REST service_host: 0.0.0.0 - service_port: 9000 + http_port: 9001 + grpc_port: 5001 name: classifier type: MODEL labels: diff --git a/executor/samples/local/metrics/model_grpc.yaml b/executor/samples/local/metrics/model_grpc.yaml deleted file mode 100644 index 265f0e78e4..0000000000 --- a/executor/samples/local/metrics/model_grpc.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1alpha2 -kind: SeldonDeployment -metadata: - labels: - app: seldon - name: seldon-model -spec: - annotations: - seldon.io/executor: "true" - name: test-deployment - predictors: - - componentSpecs: - - spec: - containers: - - image: seldonio/mock_classifier_rest:1.3 - name: classifier - graph: - children: [] - endpoint: - type: GRPC - service_host: 0.0.0.0 - service_port: 9000 - name: classifier - type: MODEL - labels: - version: v1 - name: example - replicas: 1 diff --git a/executor/samples/local/tfserving/Makefile b/executor/samples/local/tfserving/Makefile index abed50c7fa..fef38b7a38 100644 --- a/executor/samples/local/tfserving/Makefile +++ b/executor/samples/local/tfserving/Makefile @@ -2,12 +2,11 @@ BASE=../../.. ## REST -run_rest_executor: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_rest.yaml --http_port 8000 --protocol tensorflow +run_executor: + ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model.yaml --http_port 8000 --grpc_port 9500 --protocol tensorflow run_rest_executor_chain: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_rest_chain.yaml --http_port 8000 --protocol tensorflow - + ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_chain.yaml --http_port 8000 --grpc_port 9500 --protocol tensorflow run_tensorflow_serving: docker run --name tfserver --rm -p 8501:8501 -p 8500:8500 -v "${PWD}/model:/models/half_plus_two" -e MODEL_NAME=half_plus_two tensorflow/serving @@ -21,23 +20,13 @@ curl_status: curl_metadata: curl http://localhost:8000/v1/models/half_plus_two/metadata - -## GRPC - -run_grpc_executor: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_grpc.yaml --http_port 8000 --grpc_port 5000 --transport grpc --protocol tensorflow - -run_grpc_executor_chain: - ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_grpc_chain.yaml --http_port 8000 --grpc_port 5000 --transport grpc --protocol tensorflow - - grpc_test: - cd ${BASE}/proto && grpcurl -d '{"model_spec":{"name":"half_plus_two"},"inputs":{"x":{"dtype": 1, "tensor_shape": {"dim":[{"size": 3}]}, "floatVal" : [1.0, 2.0, 3.0]}}}' -plaintext -proto ./prediction_service.proto 0.0.0.0:5000 tensorflow.serving.PredictionService/Predict + cd ${BASE}/proto && grpcurl -d '{"model_spec":{"name":"half_plus_two"},"inputs":{"x":{"dtype": 1, "tensor_shape": {"dim":[{"size": 3}]}, "floatVal" : [1.0, 2.0, 3.0]}}}' -plaintext -proto ./prediction_service.proto 0.0.0.0:9500 tensorflow.serving.PredictionService/Predict grpc_status: - cd ${BASE}/proto && grpcurl -d '{"model_spec":{"name":"half_plus_two"}}' -plaintext -proto ./model_service.proto 0.0.0.0:5000 tensorflow.serving.ModelService/GetModelStatus + cd ${BASE}/proto && grpcurl -d '{"model_spec":{"name":"half_plus_two"}}' -plaintext -proto ./model_service.proto 0.0.0.0:9500 tensorflow.serving.ModelService/GetModelStatus grpc_metadata: - cd ${BASE}/proto && grpcurl -d '{"model_spec":{"name":"half_plus_two"},"metadata_field":"signature_def"}' -plaintext -proto ./prediction_service.proto 0.0.0.0:5000 tensorflow.serving.PredictionService/GetModelMetadata + cd ${BASE}/proto && grpcurl -d '{"model_spec":{"name":"half_plus_two"},"metadata_field":"signature_def"}' -plaintext -proto ./prediction_service.proto 0.0.0.0:9500 tensorflow.serving.PredictionService/GetModelMetadata diff --git a/executor/samples/local/tfserving/model_rest.yaml b/executor/samples/local/tfserving/model.yaml similarity index 90% rename from executor/samples/local/tfserving/model_rest.yaml rename to executor/samples/local/tfserving/model.yaml index 297cf89846..7e576b287b 100644 --- a/executor/samples/local/tfserving/model_rest.yaml +++ b/executor/samples/local/tfserving/model.yaml @@ -19,7 +19,8 @@ spec: endpoint: type: REST service_host: 0.0.0.0 - service_port: 8501 + http_port: 8501 + grpc_port: 8500 name: half_plus_two type: MODEL labels: diff --git a/executor/samples/local/tfserving/model_rest_chain.yaml b/executor/samples/local/tfserving/model_chain.yaml similarity index 84% rename from executor/samples/local/tfserving/model_rest_chain.yaml rename to executor/samples/local/tfserving/model_chain.yaml index 7351a96483..af7b4624a4 100644 --- a/executor/samples/local/tfserving/model_rest_chain.yaml +++ b/executor/samples/local/tfserving/model_chain.yaml @@ -18,14 +18,16 @@ spec: endpoint: type: REST service_host: 0.0.0.0 - service_port: 8501 + http_port: 8501 + grpc_port: 8500 name: half_plus_two type: MODEL children: - endpoint: type: REST service_host: 0.0.0.0 - service_port: 8501 + http_port: 8501 + grpc_port: 8500 name: half_plus_two type: MODEL labels: diff --git a/executor/samples/local/tfserving/model_grpc.yaml b/executor/samples/local/tfserving/model_grpc.yaml deleted file mode 100644 index 095b0404df..0000000000 --- a/executor/samples/local/tfserving/model_grpc.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1 -kind: SeldonDeployment -metadata: - labels: - app: seldon - name: seldon-model -spec: - annotations: - seldon.io/executor: "true" - name: test-deployment - predictors: - - componentSpecs: - - spec: - containers: - - image: tensorflow/serving:latest - name: half_plus_two - graph: - children: [] - endpoint: - type: GRPC - service_host: 0.0.0.0 - service_port: 8500 - name: half_plus_two - type: MODEL - labels: - version: v1 - name: example - replicas: 1 diff --git a/executor/samples/local/tfserving/model_grpc_chain.yaml b/executor/samples/local/tfserving/model_grpc_chain.yaml deleted file mode 100644 index ca72f5a4e1..0000000000 --- a/executor/samples/local/tfserving/model_grpc_chain.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1 -kind: SeldonDeployment -metadata: - labels: - app: seldon - name: seldon-model -spec: - annotations: - seldon.io/executor: "true" - name: test-deployment - predictors: - - componentSpecs: - - spec: - containers: - - image: tensorflow/serving:latest - name: half_plus_two - graph: - endpoint: - type: GRPC - service_host: 0.0.0.0 - service_port: 8500 - name: half_plus_two - type: MODEL - children: - - endpoint: - type: GRPC - service_host: 0.0.0.0 - service_port: 8500 - name: half_plus_two - type: MODEL - labels: - version: v1 - name: example - replicas: 1 diff --git a/executor/samples/local/tracing/Makefile b/executor/samples/local/tracing/Makefile index e1f8659f37..ce6083ec01 100644 --- a/executor/samples/local/tracing/Makefile +++ b/executor/samples/local/tracing/Makefile @@ -2,12 +2,12 @@ BASE=../../.. ## REST -run_rest_executor: - JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_rest.yaml --http_port 8000 +run_executor: + JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model.yaml --http_port 8000 --grpc_port 5000 -run_dummy_rest_model: - cd ${BASE}/../examples/models/mean_classifier && make run_rest_local +run_dummy_model: + cd ${BASE}/../examples/models/mean_classifier && make run_local curl_rest: curl -v localhost:8000/api/v0.1/predictions -H "Accept: application/json" -H "Content-Type: application/json" -d '{"data":{"ndarray":[[1.0,2.0]]}}' @@ -22,14 +22,6 @@ run_jaeger: docker run -p 6831:6831/udp -p 16686:16686 jaegertracing/all-in-one:latest -## GRPC - -run_grpc_executor: - JAEGER_SAMPLER_TYPE=const JAEGER_SAMPLER_PARAM=1 ${BASE}/executor --sdep seldon-model --namespace default --predictor example --file ./model_grpc.yaml --grpc_port 5000 --transport grpc - -run_dummy_grpc_model: - cd ${BASE}/../examples/models/mean_classifier && make run_grpc_local - grpc_test: cd ${BASE}/proto && grpcurl -d '{"data":{"ndarray":[[1.0,2.0]]}}' -plaintext -proto ./prediction.proto 0.0.0.0:5000 seldon.protos.Seldon/Predict diff --git a/executor/samples/local/metrics/model_rest.yaml b/executor/samples/local/tracing/model.yaml similarity index 91% rename from executor/samples/local/metrics/model_rest.yaml rename to executor/samples/local/tracing/model.yaml index 0bece0ef37..d09c540d85 100644 --- a/executor/samples/local/metrics/model_rest.yaml +++ b/executor/samples/local/tracing/model.yaml @@ -19,7 +19,8 @@ spec: endpoint: type: REST service_host: 0.0.0.0 - service_port: 9000 + http_port: 9001 + grpc_port: 5001 name: classifier type: MODEL labels: diff --git a/executor/samples/local/tracing/model_grpc.yaml b/executor/samples/local/tracing/model_grpc.yaml deleted file mode 100644 index 265f0e78e4..0000000000 --- a/executor/samples/local/tracing/model_grpc.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1alpha2 -kind: SeldonDeployment -metadata: - labels: - app: seldon - name: seldon-model -spec: - annotations: - seldon.io/executor: "true" - name: test-deployment - predictors: - - componentSpecs: - - spec: - containers: - - image: seldonio/mock_classifier_rest:1.3 - name: classifier - graph: - children: [] - endpoint: - type: GRPC - service_host: 0.0.0.0 - service_port: 9000 - name: classifier - type: MODEL - labels: - version: v1 - name: example - replicas: 1 diff --git a/helm-charts/seldon-abtest/templates/ab_test_1pod.json b/helm-charts/seldon-abtest/templates/ab_test_1pod.json index be2d49d8ef..2df8c248ff 100644 --- a/helm-charts/seldon-abtest/templates/ab_test_1pod.json +++ b/helm-charts/seldon-abtest/templates/ab_test_1pod.json @@ -10,10 +10,6 @@ }, "spec": { "name": "{{ .Release.Name }}", -{{- if .Values.oauth.key }} - "oauth_key": "{{ .Values.oauth.key }}", - "oauth_secret": "{{ .Values.oauth.secret }}", -{{- end }} "predictors": [ { "name": "{{ .Values.predictor.name }}", @@ -22,7 +18,7 @@ "spec": { "containers": [ { - "image": "{{ .Values.modela.image.name }}_{{ lower .Values.protocol }}:{{ .Values.modela.image.version }}", + "image": "{{ .Values.modela.image.name }}:{{ .Values.modela.image.version }}", "imagePullPolicy": "IfNotPresent", "name": "{{ .Values.modela.name }}", "resources": { @@ -32,7 +28,7 @@ } }, { - "image": "{{ .Values.modelb.image.name }}_{{ lower .Values.protocol }}:{{ .Values.modelb.image.version }}", + "image": "{{ .Values.modelb.image.name }}:{{ .Values.modelb.image.version }}", "imagePullPolicy": "IfNotPresent", "name": "{{ .Values.modelb.name }}", "resources": { @@ -47,9 +43,6 @@ }], "graph": { "name": "{{ .Release.Name }}", - "endpoint":{ - "type":"{{ .Values.protocol }}" - }, "implementation":"RANDOM_ABTEST", "parameters": [ { @@ -61,17 +54,11 @@ "children": [ { "name": "{{ .Values.modela.name }}", - "endpoint":{ - "type":"{{ .Values.protocol }}" - }, "type":"MODEL", "children":[] }, { "name": "{{ .Values.modelb.name }}", - "endpoint":{ - "type":"{{ .Values.protocol }}" - }, "type":"MODEL", "children":[] } diff --git a/helm-charts/seldon-abtest/templates/ab_test_2pods.json b/helm-charts/seldon-abtest/templates/ab_test_2pods.json index 95d20bdcc2..34baf69a3f 100644 --- a/helm-charts/seldon-abtest/templates/ab_test_2pods.json +++ b/helm-charts/seldon-abtest/templates/ab_test_2pods.json @@ -10,10 +10,6 @@ }, "spec": { "name": "{{ .Release.Name }}", -{{- if .Values.oauth.key }} - "oauth_key": "{{ .Values.oauth.key }}", - "oauth_secret": "{{ .Values.oauth.secret }}", -{{- end }} "predictors": [ { "name": "{{ .Values.predictor.name }}", @@ -22,7 +18,7 @@ "spec": { "containers": [ { - "image": "{{ .Values.modela.image.name }}_{{ lower .Values.protocol }}:{{ .Values.modela.image.version }}", + "image": "{{ .Values.modela.image.name }}:{{ .Values.modela.image.version }}", "imagePullPolicy": "IfNotPresent", "name": "{{ .Values.modela.name }}", "resources": { @@ -42,7 +38,7 @@ "spec":{ "containers":[ { - "image": "{{ .Values.modela.image.name }}_{{ lower .Values.protocol }}:{{ .Values.modela.image.version }}", + "image": "{{ .Values.modela.image.name }}:{{ .Values.modela.image.version }}", "imagePullPolicy": "IfNotPresent", "name": "{{ .Values.modelb.name }}", "resources": { @@ -57,9 +53,6 @@ }], "graph": { "name": "{{ .Release.Name }}", - "endpoint":{ - "type":"{{ .Values.protocol }}" - }, "implementation":"RANDOM_ABTEST", "parameters": [ { @@ -71,17 +64,11 @@ "children": [ { "name": "{{ .Values.modela.name }}", - "endpoint":{ - "type":"{{ .Values.protocol }}" - }, "type":"MODEL", "children":[] }, { "name": "{{ .Values.modelb.name }}", - "endpoint":{ - "type":"{{ .Values.protocol }}" - }, "type":"MODEL", "children":[] } diff --git a/helm-charts/seldon-abtest/values.yaml b/helm-charts/seldon-abtest/values.yaml index 9746f8b08e..dc8fe7b293 100644 --- a/helm-charts/seldon-abtest/values.yaml +++ b/helm-charts/seldon-abtest/values.yaml @@ -1,21 +1,16 @@ -protocol: REST separate_pods: true modela: image: name: seldonio/mock_classifier - version: 1.3 + version: 1.5.0-dev name: classifier-1 # resources: { "requests": { "memory": "1Mi" }} modelb: image: name: seldonio/mock_classifier - version: 1.3 + version: 1.5.0-dev name: classifier-2 traffic_modela_percentage: 0.5 replicas: 1 -# Add oauth key and secret if using the default API Oauth Gateway for ingress -oauth: - key: - secret: predictor: name: default diff --git a/helm-charts/seldon-core-operator/values.yaml b/helm-charts/seldon-core-operator/values.yaml index 2ea04049d9..7df7f462cf 100644 --- a/helm-charts/seldon-core-operator/values.yaml +++ b/helm-charts/seldon-core-operator/values.yaml @@ -102,50 +102,39 @@ predictiveUnit: defaultEnvSecretRefName: "" predictor_servers: MLFLOW_SERVER: - grpc: - defaultImageVersion: "1.5.0-dev" - image: seldonio/mlflowserver_grpc - rest: - defaultImageVersion: "1.5.0-dev" - image: seldonio/mlflowserver_rest + protocols: + seldon: + defaultImageVersion: "1.5.0-dev" + image: seldonio/mlflowserver SKLEARN_SERVER: - grpc: - defaultImageVersion: "1.5.0-dev" - image: seldonio/sklearnserver_grpc - rest: - defaultImageVersion: "1.5.0-dev" - image: seldonio/sklearnserver_rest protocols: + seldon: + defaultImageVersion: "1.5.0-dev" + image: seldonio/sklearnserver kfserving: defaultImageVersion: "0.1.1" image: seldonio/mlserver TENSORFLOW_SERVER: - grpc: - defaultImageVersion: "1.5.0-dev" - image: seldonio/tfserving-proxy_grpc - rest: - defaultImageVersion: "1.5.0-dev" - image: seldonio/tfserving-proxy_rest - tensorflow: true - tfImage: tensorflow/serving:2.1.0 + protocols: + seldon: + defaultImageVersion: "1.5.0-dev" + image: seldonio/tfserving-proxy + tensorflow: + defaultImageVersion: 2.1.0 + image: tensorflow/serving XGBOOST_SERVER: - grpc: - defaultImageVersion: "1.5.0-dev" - image: seldonio/xgboostserver_grpc - rest: - defaultImageVersion: "1.5.0-dev" - image: seldonio/xgboostserver_rest protocols: + seldon: + defaultImageVersion: "1.5.0-dev" + image: seldonio/xgboostserver kfserving: defaultImageVersion: "0.1.1" image: seldonio/mlserver TRITON_SERVER: - grpc: - defaultImageVersion: "20.08-py3" - image: nvcr.io/nvidia/tritonserver - rest: - defaultImageVersion: "20.08-py3" - image: nvcr.io/nvidia/tritonserver + protocols: + kfserving: + defaultImageVersion: "20.08-py3" + image: nvcr.io/nvidia/tritonserver # ## Other # You can choose the crds to not be installed if you already installed them diff --git a/helm-charts/seldon-mab/templates/mab.json b/helm-charts/seldon-mab/templates/mab.json index 142f5c4c22..97a2ce2cd4 100644 --- a/helm-charts/seldon-mab/templates/mab.json +++ b/helm-charts/seldon-mab/templates/mab.json @@ -7,10 +7,6 @@ }, "spec": { "name": "{{ .Release.Name }}", -{{- if .Values.oauth.key }} - "oauth_key": "{{ .Values.oauth.key }}", - "oauth_secret": "{{ .Values.oauth.secret }}", -{{- end }} "predictors": [ { "name": "{{ .Values.predictor.name }}", @@ -19,7 +15,7 @@ "spec": { "containers": [ { - "image": "{{ .Values.modela.image.name }}_{{ lower .Values.protocol }}:{{ .Values.modela.image.version }}", + "image": "{{ .Values.modela.image.name }}:{{ .Values.modela.image.version }}", "imagePullPolicy": "IfNotPresent", "name": "{{ .Values.modela.name }}", "resources": { @@ -34,7 +30,7 @@ "spec":{ "containers":[ { - "image": "{{ .Values.modelb.image.name }}_{{ lower .Values.protocol }}:{{ .Values.modelb.image.version }}", + "image": "{{ .Values.modelb.image.name }}:{{ .Values.modelb.image.version }}", "imagePullPolicy": "IfNotPresent", "name": "{{ .Values.modelb.name }}", "resources": { @@ -50,7 +46,7 @@ { "spec":{ "containers": [{ - "image": "{{ .Values.mab.image.name }}_{{ lower .Values.protocol }}:{{ .Values.mab.image.version }}", + "image": "{{ .Values.mab.image.name }}:{{ .Values.mab.image.version }}", "name": "{{ .Values.mab.name }}" }], "terminationGracePeriodSeconds": 20 @@ -58,9 +54,6 @@ ], "graph": { "name": "{{ .Values.mab.name }}", - "endpoint": { - "type": "{{ .Values.protocol }}" - }, "type":"ROUTER", "parameters": [ { @@ -82,17 +75,11 @@ "children": [ { "name": "{{ .Values.modela.name }}", - "endpoint":{ - "type": "{{ .Values.protocol }}" - }, "type":"MODEL", "children":[] }, { "name": "{{ .Values.modelb.name }}", - "endpoint":{ - "type": "{{ .Values.protocol }}" - }, "type":"MODEL", "children":[] } diff --git a/helm-charts/seldon-mab/values.yaml b/helm-charts/seldon-mab/values.yaml index c842585aa1..824716ab03 100644 --- a/helm-charts/seldon-mab/values.yaml +++ b/helm-charts/seldon-mab/values.yaml @@ -1,35 +1,28 @@ -protocol: REST modela: image: name: seldonio/mock_classifier - version: 1.3 - endpoint: REST + version: 1.5.0-dev name: classifier-1 modelb: image: name: seldonio/mock_classifier - version: 1.3 - endpoint: REST + version: 1.5.0-dev name: classifier-2 mab: image: name: seldonio/mab_epsilon_greedy - version: 1.3 + version: 1.5.0-dev name: eg-router branches: 2 epsilon: 0.2 verbose: 1 replicas: 1 -# Add oauth key and secret if using the default API Oauth Gateway for ingress -oauth: - key: - secret: sdepLabels: app: "seldon" predictorLabels: - version: "v1" + version: 1.5.0-dev fluentd: "true" engine: diff --git a/helm-charts/seldon-single-model/templates/seldondeployment.json b/helm-charts/seldon-single-model/templates/seldondeployment.json index 8bcd4d1666..c87da60d3e 100644 --- a/helm-charts/seldon-single-model/templates/seldondeployment.json +++ b/helm-charts/seldon-single-model/templates/seldondeployment.json @@ -13,7 +13,6 @@ "spec": { "name": "{{ .Release.Name }}", "protocol": "{{ .Values.protocol }}", - "transport": "{{ .Values.transport }}", "annotations": { {{- range $name, $val := .Values.annotations }} "{{ $name }}": "{{ $val }}", diff --git a/helm-charts/seldon-single-model/values.yaml b/helm-charts/seldon-single-model/values.yaml index 57b8b7903c..4d1b53e1c7 100644 --- a/helm-charts/seldon-single-model/values.yaml +++ b/helm-charts/seldon-single-model/values.yaml @@ -2,7 +2,6 @@ apiVersion: machinelearning.seldon.io/v1 protocol: seldon -transport: rest model: # model.implementation -- Implementation of Prepackaged Model Server diff --git a/incubating/wrappers/s2i/java-jni/Makefile b/incubating/wrappers/s2i/java-jni/Makefile index 0ca61c0f97..91327d914b 100644 --- a/incubating/wrappers/s2i/java-jni/Makefile +++ b/incubating/wrappers/s2i/java-jni/Makefile @@ -1,6 +1,6 @@ SHELL := /bin/bash SELDON_CORE_DIR = ../../../.. -VERSION := 0.3.0 +VERSION := 0.4.0 IMAGE_REGISTRY = docker.io/seldonio IMAGE_NAME_BUILD = ${IMAGE_REGISTRY}/s2i-java-jni-build:${VERSION} IMAGE_NAME_RUNTIME = ${IMAGE_REGISTRY}/s2i-java-jni-runtime:${VERSION} diff --git a/incubating/wrappers/s2i/java-jni/environment_grpc b/incubating/wrappers/s2i/java-jni/environment_grpc deleted file mode 100644 index 6339f452c6..0000000000 --- a/incubating/wrappers/s2i/java-jni/environment_grpc +++ /dev/null @@ -1,6 +0,0 @@ -MODEL_NAME=JavaJNIServer -API_TYPE=GRPC -SERVICE_TYPE=MODEL -PERSISTENCE=0 -CONDA_ENV_NAME=java-jni -PAYLOAD_PASSTHROUGH=true diff --git a/incubating/wrappers/s2i/java-jni/environment_rest b/incubating/wrappers/s2i/java-jni/environment_rest deleted file mode 100644 index 855c4fa6de..0000000000 --- a/incubating/wrappers/s2i/java-jni/environment_rest +++ /dev/null @@ -1,6 +0,0 @@ -MODEL_NAME=JavaJNIServer -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 -CONDA_ENV_NAME=java-jni -PAYLOAD_PASSTHROUGH=true diff --git a/incubating/wrappers/s2i/java-jni/s2i/bin/assemble b/incubating/wrappers/s2i/java-jni/s2i/bin/assemble index dba5eeea18..13f18cdf8d 100755 --- a/incubating/wrappers/s2i/java-jni/s2i/bin/assemble +++ b/incubating/wrappers/s2i/java-jni/s2i/bin/assemble @@ -12,13 +12,6 @@ if [[ "$1" == "-h" ]]; then exec /usr/libexec/s2i/usage fi - -if [[ -z "$API_TYPE" ]]; then - - echo "Failed to find required env var API_TYPE, should be either REST or GRPC." - exit 1 -fi - if [[ -z "$SERVICE_TYPE" ]]; then echo "Failed to find required env var SERVICE_TYPE, should be one of MODEL, ROUTER, TRANSFORMER, COMBINER." diff --git a/incubating/wrappers/s2i/java-jni/s2i/bin/run b/incubating/wrappers/s2i/java-jni/s2i/bin/run index d1fc66d3fd..e53e72da25 100755 --- a/incubating/wrappers/s2i/java-jni/s2i/bin/run +++ b/incubating/wrappers/s2i/java-jni/s2i/bin/run @@ -8,9 +8,9 @@ # # Check environment vars -if [[ -z "$JAVA_IMPORT_PATH" || -z "$API_TYPE" || -z "$SERVICE_TYPE" || -z "$PERSISTENCE" ]]; then +if [[ -z "$JAVA_IMPORT_PATH" || -z "$SERVICE_TYPE" || -z "$PERSISTENCE" ]]; then - echo "Failed to find required env vars JAVA_IMPORT_PATH, API_TYPE, SERVICE_TYPE, PERSISTENCE" + echo "Failed to find required env vars JAVA_IMPORT_PATH, SERVICE_TYPE, PERSISTENCE" exit 1 fi @@ -27,7 +27,6 @@ exec \ env JAVA_IMPORT_PATH="$JAVA_IMPORT_PATH" \ seldon-core-microservice \ $MODEL_NAME \ - $API_TYPE \ --service-type $SERVICE_TYPE \ --persistence $PERSISTENCE diff --git a/integrations/tfserving/Makefile b/integrations/tfserving/Makefile deleted file mode 100644 index 09e39ac69b..0000000000 --- a/integrations/tfserving/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -SHELL := /bin/bash -VERSION := $(shell cat ../../version.txt) -IMAGE_NAME_BASE=tfserving-proxy -IMAGE_BASE=seldonio/${IMAGE_NAME_BASE} -KIND_NAME ?= kind - -build_%: - s2i build \ - -E environment_$* \ - . \ - seldonio/seldon-core-s2i-python37-ubi8:${VERSION} \ - ${IMAGE_BASE}_$*:${VERSION} - -push_%: - docker push ${IMAGE_BASE}_$*:${VERSION} - -kind_load_%: - kind load -v 3 docker-image ${IMAGE_BASE}_$*:${VERSION} --name ${KIND_NAME} - -.PHONY: build_all -build_all: build_rest build_grpc - -.PHONY: push_all -push_all: push_rest push_grpc - -.PHONY: kind_load_all -kind_load: build_all kind_load_rest kind_load_grpc - -# https://connect.redhat.com/project/4098981/view -scan_rest=ospid-2cbfde5f-10d2-4cc1-9b1d-0dff7f6d1021 -# https://connect.redhat.com/project/4100321/view -scan_grpc=ospid-da09f299-64d1-4e19-86c9-af60a0ddb851 -redhat-image-scan-%: - docker pull ${IMAGE_BASE}_$*:${VERSION} - source ~/.config/seldon/seldon-core/redhat-image-passwords.sh && \ - echo $${rh_password_tfproxy_$*} | docker login -u unused scan.connect.redhat.com --password-stdin - docker tag ${IMAGE_BASE}_$*:${VERSION} scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} - docker push scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} - -.PHONY: redhat-image-scan -redhat-image-scan: redhat-image-scan-rest redhat-image-scan-grpc diff --git a/notebooks/backwards_compatability.ipynb b/notebooks/backwards_compatability.ipynb new file mode 100644 index 0000000000..df99712059 --- /dev/null +++ b/notebooks/backwards_compatability.ipynb @@ -0,0 +1,1132 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Backwards Compatability Examples with Different Protocols\n", + "\n", + "## Prerequisites\n", + "\n", + " * A kubernetes cluster with kubectl configured\n", + " * curl\n", + " * grpcurl\n", + " * pygmentize\n", + " \n", + "\n", + "## Setup Seldon Core\n", + "\n", + "Use the setup notebook to [Setup Cluster](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html) to setup Seldon Core with an ingress - either Ambassador or Istio.\n", + "\n", + "Then port-forward to that ingress on localhost:8003 in a separate terminal either with:\n", + "\n", + " * Ambassador: `kubectl port-forward $(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080`\n", + " * Istio: `kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8003:80`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], + "source": [ + "!kubectl create namespace seldon" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], + "source": [ + "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import time" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "VERSION=!cat ../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model with Old REST Wrapper Upgraded" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will deploy a REST model that uses the SELDON Protocol namely by specifying the attribute `protocol: seldon`" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "%%writetemplate resources/model_seldon.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: example-seldon\n", + "spec:\n", + " protocol: seldon\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier_rest:1.4.0\n", + " name: classifier\n", + " graph:\n", + " name: classifier\n", + " type: MODEL\n", + " name: model\n", + " replicas: 1" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-seldon created\r\n" + ] + } + ], + "source": [ + "!kubectl apply -f resources/model_seldon.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-seldon-model-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-seldon-model-0-classifier\" successfully rolled out\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-seldon -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], + "source": [ + "for i in range(60):\n", + " state=!kubectl get sdep example-seldon -o jsonpath='{.status.state}'\n", + " state=state[0]\n", + " print(state)\n", + " if state==\"Available\":\n", + " break\n", + " time.sleep(1)\n", + "assert(state==\"Available\")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/example-seldon/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "%%writetemplate resources/model_seldon.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: example-seldon\n", + "spec:\n", + " protocol: seldon\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " name: classifier\n", + " graph:\n", + " name: classifier\n", + " type: MODEL\n", + " name: model\n", + " replicas: 1" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-seldon configured\r\n" + ] + } + ], + "source": [ + "!kubectl apply -f resources/model_seldon.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-seldon-model-0-classifier\" rollout to finish: 1 old replicas are pending termination...\n", + "Waiting for deployment \"example-seldon-model-0-classifier\" rollout to finish: 1 old replicas are pending termination...\n", + "deployment \"example-seldon-model-0-classifier\" successfully rolled out\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-seldon -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], + "source": [ + "for i in range(60):\n", + " state=!kubectl get sdep example-seldon -o jsonpath='{.status.state}'\n", + " state=state[0]\n", + " print(state)\n", + " if state==\"Available\":\n", + " break\n", + " time.sleep(1)\n", + "assert(state==\"Available\")" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/example-seldon/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}}\n" + ] + } + ], + "source": [ + "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[1.0,2.0,5.0]]}}' \\\n", + " -rpc-header seldon:example-seldon -rpc-header namespace:seldon \\\n", + " -plaintext \\\n", + " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example-seldon\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f resources/model_seldon.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Model with Old GRPC Wrapper Upgraded" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We will deploy a gRPC model that uses the SELDON Protocol namely by specifying the attribute `protocol: seldon`" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting resources/model_seldon_grpc.yaml\n" + ] + } + ], + "source": [ + "%%writefile resources/model_seldon_grpc.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: grpc-seldon\n", + "spec:\n", + " name: grpcseldon\n", + " protocol: seldon\n", + " transport: grpc\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier_grpc:1.3\n", + " name: classifier\n", + " graph:\n", + " name: classifier\n", + " type: MODEL\n", + " endpoint:\n", + " type: GRPC\n", + " name: model\n", + " replicas: 1" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/grpc-seldon created\r\n" + ] + } + ], + "source": [ + "!kubectl apply -f resources/model_seldon_grpc.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"grpc-seldon-model-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"grpc-seldon-model-0-classifier\" successfully rolled out\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=grpc-seldon -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], + "source": [ + "for i in range(60):\n", + " state=!kubectl get sdep grpc-seldon -o jsonpath='{.status.state}'\n", + " state=state[0]\n", + " print(state)\n", + " if state==\"Available\":\n", + " break\n", + " time.sleep(1)\n", + "assert(state==\"Available\")" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'names': ['proba'], 'ndarray': [[0.6418340450887311]]}}\n" + ] + } + ], + "source": [ + "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[1.0,2.0,5.0,6.0]]}}' \\\n", + " -rpc-header seldon:grpc-seldon -rpc-header namespace:seldon \\\n", + " -plaintext \\\n", + " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"grpc-seldon\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f resources/model_seldon_grpc.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Old Operator and Model Upgraded" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"seldon\" uninstalled\r\n", + "Error: uninstall: Release not loaded: seldon-core-operator: release: not found\r\n" + ] + } + ], + "source": [ + "!helm delete seldon seldon-core-operator \\\n", + " --namespace seldon-system " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: seldon\r\n", + "LAST DEPLOYED: Mon Nov 9 08:57:55 2020\r\n", + "NAMESPACE: seldon-system\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], + "source": [ + "!helm install seldon seldon-core-operator \\\n", + " --repo https://storage.googleapis.com/seldon-charts \\\n", + " --version 1.4.0 \\\n", + " --namespace seldon-system \\\n", + " --wait" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "%%writetemplate resources/model_seldon.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: example-seldon\n", + "spec:\n", + " protocol: seldon\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier_rest:1.4.0\n", + " name: classifier\n", + " graph:\n", + " name: classifier\n", + " type: MODEL\n", + " name: model\n", + " replicas: 1" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting ../servers/sklearnserver/samples/iris.yaml\n" + ] + } + ], + "source": [ + "%%writefile ../servers/sklearnserver/samples/iris.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: sklearn\n", + "spec:\n", + " name: iris\n", + " predictors:\n", + " - graph:\n", + " children: []\n", + " implementation: SKLEARN_SERVER\n", + " modelUri: gs://seldon-models/sklearn/iris\n", + " name: classifier\n", + " name: default\n", + " replicas: 1\n", + " svcOrchSpec: \n", + " env: \n", + " - name: SELDON_LOG_LEVEL\n", + " value: DEBUG" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-seldon created\r\n" + ] + } + ], + "source": [ + "!kubectl apply -f resources/model_seldon.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/sklearn created\r\n" + ] + } + ], + "source": [ + "!kubectl apply -f ../servers/sklearnserver/samples/iris.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-seldon-model-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-seldon-model-0-classifier\" successfully rolled out\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-seldon -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"sklearn-default-0-classifier\" successfully rolled out\r\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=sklearn -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], + "source": [ + "for i in range(60):\n", + " state=!kubectl get sdep example-seldon -o jsonpath='{.status.state}'\n", + " state=state[0]\n", + " print(state)\n", + " if state==\"Available\":\n", + " break\n", + " time.sleep(1)\n", + "assert(state==\"Available\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/example-seldon/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['t:0', 't:1', 't:2'], 'ndarray': [[9.912315378486718e-07, 0.0007015931307743852, 0.9992974156376878]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0, 6.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/sklearn/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Release \"seldon\" has been upgraded. Happy Helming!\r\n", + "NAME: seldon\r\n", + "LAST DEPLOYED: Mon Nov 9 09:06:11 2020\r\n", + "NAMESPACE: seldon-system\r\n", + "STATUS: deployed\r\n", + "REVISION: 2\r\n", + "TEST SUITE: None\r\n" + ] + } + ], + "source": [ + "!helm upgrade seldon \\\n", + " ../helm-charts/seldon-core-operator \\\n", + " --namespace seldon-system \\\n", + " --wait" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"example-seldon-model-0-classifier\" successfully rolled out\r\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-seldon -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"sklearn-default-0-classifier\" successfully rolled out\r\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=sklearn -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], + "source": [ + "for i in range(60):\n", + " state=!kubectl get sdep example-seldon -o jsonpath='{.status.state}'\n", + " state=state[0]\n", + " print(state)\n", + " if state==\"Available\":\n", + " break\n", + " time.sleep(1)\n", + "assert(state==\"Available\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Only REST calls will be available as image is still old python wrapper" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/example-seldon/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Rest and gRPC calls will work with new server as image will have been updated." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['t:0', 't:1', 't:2'], 'ndarray': [[9.912315378486718e-07, 0.0007015931307743852, 0.9992974156376878]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0, 6.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/sklearn/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'names': ['t:0', 't:1', 't:2'], 'ndarray': [[9.912315378486718e-07, 0.0007015931307743852, 0.9992974156376878]]}}\n" + ] + } + ], + "source": [ + "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[1.0,2.0,5.0,6.0]]}}' \\\n", + " -rpc-header seldon:sklearn -rpc-header namespace:seldon \\\n", + " -plaintext \\\n", + " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "%%writetemplate resources/model_seldon.yaml\n", + "apiVersion: machinelearning.seldon.io/v1\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " name: example-seldon\n", + "spec:\n", + " protocol: seldon\n", + " predictors:\n", + " - componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " name: classifier\n", + " graph:\n", + " name: classifier\n", + " type: MODEL\n", + " name: model\n", + " replicas: 1" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-seldon configured\r\n" + ] + } + ], + "source": [ + "!kubectl apply -f resources/model_seldon.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-seldon-model-0-classifier\" rollout to finish: 1 old replicas are pending termination...\n", + "Waiting for deployment \"example-seldon-model-0-classifier\" rollout to finish: 1 old replicas are pending termination...\n", + "deployment \"example-seldon-model-0-classifier\" successfully rolled out\n" + ] + } + ], + "source": [ + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-seldon -o jsonpath='{.items[0].metadata.name}')" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], + "source": [ + "for i in range(60):\n", + " state=!kubectl get sdep example-seldon -o jsonpath='{.status.state}'\n", + " state=state[0]\n", + " print(state)\n", + " if state==\"Available\":\n", + " break\n", + " time.sleep(1)\n", + "assert(state==\"Available\")" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/example-seldon/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}}\n" + ] + } + ], + "source": [ + "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[1.0,2.0,5.0]]}}' \\\n", + " -rpc-header seldon:example-seldon -rpc-header namespace:seldon \\\n", + " -plaintext \\\n", + " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example-seldon\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f resources/model_seldon.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"sklearn\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f ../servers/sklearnserver/samples/iris.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebooks/explainer_examples.ipynb b/notebooks/explainer_examples.ipynb index d7482be7b9..e952f28f13 100644 --- a/notebooks/explainer_examples.ipynb +++ b/notebooks/explainer_examples.ipynb @@ -79,7 +79,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "namespace/seldon created\r\n" + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" ] } ], @@ -123,7 +123,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -159,7 +159,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 4, "metadata": { "scrolled": true }, @@ -178,14 +178,15 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "deployment \"income-default-0-classifier\" successfully rolled out\r\n" + "Waiting for deployment \"income-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"income-default-0-classifier\" successfully rolled out\n" ] } ], @@ -195,7 +196,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 6, "metadata": { "scrolled": true }, @@ -204,7 +205,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "deployment \"income-default-explainer\" successfully rolled out\r\n" + "Waiting for deployment \"income-default-explainer\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"income-default-explainer\" successfully rolled out\n" ] } ], @@ -214,7 +216,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 7, "metadata": { "scrolled": true }, @@ -234,7 +236,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 8, "metadata": { "scrolled": true }, @@ -262,7 +264,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -288,7 +290,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 10, "metadata": { "scrolled": true }, @@ -316,7 +318,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -325,10 +327,9 @@ "text": [ " % Total % Received % Xferd Average Speed Time Time Time Current\n", " Dload Upload Total Spent Left Speed\n", - "100 2342 100 2251 100 91 9146 369 --:--:-- --:--:-- --:--:-- 9150\n", + "100 1715 100 1624 100 91 8837 495 --:--:-- --:--:-- --:--:-- 8826\n", "\u001b[1;39m[\n", - " \u001b[0;32m\"Marital Status = Separated\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"Capital Gain <= 0.00\"\u001b[0m\u001b[1;39m\n", + " \u001b[0;32m\"Marital Status = Separated\"\u001b[0m\u001b[1;39m\n", "\u001b[1;39m]\u001b[0m\n" ] } @@ -341,7 +342,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 12, "metadata": { "scrolled": true }, @@ -369,7 +370,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 13, "metadata": {}, "outputs": [ { @@ -404,7 +405,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 14, "metadata": { "scrolled": true }, @@ -423,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 15, "metadata": { "scrolled": true }, @@ -443,7 +444,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "metadata": {}, "outputs": [ { @@ -460,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 17, "metadata": { "scrolled": true }, @@ -473,7 +474,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -492,7 +493,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 19, "metadata": { "scrolled": true }, @@ -527,7 +528,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -549,7 +550,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 21, "metadata": { "scrolled": true }, @@ -570,7 +571,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 22, "metadata": { "scrolled": true }, @@ -587,291 +588,6 @@ "!kubectl delete -f resources/moviesentiment_explainer.yaml" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Anchor Imake Explanations on an Imagenet Model\n", - "\n", - "The model and explainer used here can be trained yourself following the full example in the [Anchor explanations for ImageNet](https://docs.seldon.io/projects/alibi/en/latest/examples/anchor_image_imagenet.html) in the Alibi project documentation.\n", - "\n", - "Note we used a python3.6 and version 0.5.2 of Alibi.\n" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Overwriting resources/imagenet_explainer_grpc.yaml\n" - ] - } - ], - "source": [ - "%%writefile resources/imagenet_explainer_grpc.yaml\n", - "apiVersion: machinelearning.seldon.io/v1\n", - "kind: SeldonDeployment\n", - "metadata:\n", - " name: image\n", - "spec:\n", - " annotations:\n", - " seldon.io/rest-timeout: \"10000000\"\n", - " seldon.io/grpc-timeout: \"10000000\"\n", - " seldon.io/grpc-max-message-size: \"1000000000\"\n", - " name: image\n", - " predictors:\n", - " - componentSpecs:\n", - " - spec:\n", - " containers:\n", - " - image: docker.io/seldonio/imagenet-transformer:0.1\n", - " name: transformer\n", - " graph:\n", - " name: transformer\n", - " type: TRANSFORMER\n", - " endpoint:\n", - " type: GRPC\n", - " children: \n", - " - implementation: TENSORFLOW_SERVER\n", - " modelUri: gs://seldon-models/tfserving/imagenet/model\n", - " name: classifier\n", - " endpoint:\n", - " type: GRPC\n", - " parameters:\n", - " - name: model_name\n", - " type: STRING\n", - " value: classifier\n", - " - name: model_input\n", - " type: STRING\n", - " value: input_image\n", - " - name: model_output\n", - " type: STRING\n", - " value: predictions/Softmax:0\n", - " svcOrchSpec:\n", - " resources:\n", - " requests:\n", - " memory: 10Gi\n", - " limits:\n", - " memory: 10Gi \n", - " env:\n", - " - name: SELDON_LOG_LEVEL\n", - " value: DEBUG\n", - " explainer:\n", - " type: AnchorImages\n", - " modelUri: gs://seldon-models/tfserving/imagenet/explainer-py36-0.5.2\n", - " config:\n", - " batch_size: \"100\"\n", - " endpoint:\n", - " type: GRPC\n", - " name: default\n", - " replicas: 1" - ] - }, - { - "cell_type": "code", - "execution_count": 42, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seldondeployment.machinelearning.seldon.io/image created\r\n" - ] - } - ], - "source": [ - "!kubectl apply -f resources/imagenet_explainer_grpc.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": 43, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Waiting for deployment \"image-default-0-transformer-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", - "deployment \"image-default-0-transformer-classifier\" successfully rolled out\n" - ] - } - ], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=image -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": 44, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "deployment \"image-default-explainer\" successfully rolled out\r\n" - ] - } - ], - "source": [ - "!kubectl rollout status deploy/image-default-explainer" - ] - }, - { - "cell_type": "code", - "execution_count": 45, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from PIL import Image\n", - "import matplotlib\n", - "%matplotlib inline\n", - "import matplotlib.pyplot as plt\n", - "from tensorflow.keras.applications.inception_v3 import InceptionV3, decode_predictions\n", - "import alibi\n", - "from alibi.datasets import fetch_imagenet\n", - "import numpy as np\n", - "\n", - "def get_image_data():\n", - " data = []\n", - " image_shape = (299, 299, 3)\n", - " target_size = image_shape[:2]\n", - " image = Image.open(\"cat-raw.jpg\").convert('RGB')\n", - " image = np.expand_dims(image.resize(target_size), axis=0)\n", - " data.append(image)\n", - " data = np.concatenate(data, axis=0)\n", - " return data\n", - "\n", - "data = get_image_data()" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "import numpy as np\n", - "sc = SeldonClient(\n", - " deployment_name=\"image\",\n", - " namespace=\"seldon\",\n", - " grpc_max_send_message_length= 27 * 1024 * 1024,\n", - " grpc_max_receive_message_length= 27 * 1024 * 1024, \n", - " gateway=\"ambassador\",\n", - " transport=\"grpc\",\n", - " gateway_endpoint=\"localhost:8003\",\n", - " client_return_type='proto')" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 47, - "metadata": {}, - "output_type": "execute_result" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAARUAAAEICAYAAABxpmCnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOy9ebBlSV7f9/ll5jl3ee9VVVf3dM9MNzA2m1gCGAcMkoJFChAGjIwxBksgaZAhkAnhUYBsB8JGHtnCDi1GNgIkoUAsskBIRgiB2STQDBJIBiRgAsQOPWaA2Xu6uqrevfeczJ//+P3ynLz3vVdLd1dNjaYy4r5378mz5sn85u/3/S0pqsrD8rA8LA/Li1XCu/sGHpaH5WH596s8BJWH5WF5WF7U8hBUHpaH5WF5UctDUHlYHpaH5UUtD0HlYXlYHpYXtTwElYflYXlYXtTyEFQeloflYXlRywsCFRFREbkhIl99sP3HReSVL+zWHpZ3dxGRjxeRX35338fDcn+LiCxE5JdE5CXNtk8WkesiUkTkk295AlV93h9AgQ842PaHgR9sfr8WeO0dnk+AvwS8wz9/CRCv+yDge4C3Ae8Efgj44ObYD/dtb7fH2jvvAvgm4I3Ac8DPAp/W1H8o8NPAM/75Z8CHNvVfBvwGcA34HeCvAamp//3AT/q53wB83AXP93cO2wx4GnjFnbZ38/11wAa47s/8j4CXvZD3+Z708ef/oova5w6O/yjg3wA3/f9H3Wb/PwL8InAD+HXg45u6LwJ+zd/FDwIvb+quAN8KvNU/rz047z/3Pn0N+DngM5u6r/Rz1s8pUIDHvP4XDupH4Hvv9BmB/wj4MT/2LcCfaer+e+B/P6cdngY++ZZt9QJf7Hmg8v8An9/8fu1hQ97ifH8K+GXgKeBJ4N8B/7XXvQr4QuAq0AH/C/BLzbEf7PWfedi5gCO/j1dg0tlnOAC8onnxr8BALQKvAd7QHP/+wBX/fhX4UeDLm9/vAD7Hj/1jGDA9cnAPHwe8/rDNeGGg8kUH9/T3n8c7jC+kD7y7PrwAUAF6bIL5MmzCeY3/7i/Y/w95/e/1/vMk8KTX/QEMLD7Mz/s3gNc3x34z8A+BtfexXwf+ZFP/EfgEBXys98tzJwfvwz96QZ0Avwn8iTt5RuAxv+/P9/oT4EOa8z2FTVaLg+s8zf0EFX+QU+Cpg4Z4bfMC3gT8WX+g3z1o4J8Avrj5/YXAv77g2lf9+o8ebP+AO+lcmETx2edsT8CfBm5ecNyjmCTzDf77M4BfONjnV4AvPDjnz3gHetFBxX//aeDn/fvvAf4pJtH9MvC5zX7f4h3/+7FZ95OBT8cA/Dngt4H/tn1fzbFfgQ2K53z/z2rqvgD4l8BfxUD1N2mkwVs801Vs4P2OH/ePffsjwPdhs/gz/v0pr/tqIDNLal932D63uean+HNKs+3/Az71gv1/on2fB3V/Ffj65vfL/R2/v/9+O/AxTf1XAv/ignO9yp/pVefUCSYtv/qCYz/R38vRnTwj8L8Cf/c27fSrwCcebHua24DKi03UfiBQVPVNdYOqvlZVX9vs81LgMob2Xwh8vYg84nUfhomAtfycbzuvfALwZlV9x93epIg8galTv3Cw/V3YS/3rWKO3dZ8nItewTvKRwN9qqw8vgaljtXwZ8GOq+obDe1HVV6jq03dy36p6eJ16b48Bnw38jIgcYYDy7cDjmNj+DSLyoc0hn4cNzBMMCL4J+FOqeuL3/aMX3MKvAx+Pvb+/APxfIvKypv5jMRB7DPjLwDeJyLn33JS/i83iH+b3+9d8e8DA5v2A98Umq6/zdvgfgH8BfKmqHqvql96qfc4pH4ZJom3g2xs4p6+JSAQ+GniJiPyaiLxJRL5ORFbtbud8//Bb1Ld1iMj3icgG+H+xyeKnz7nnj8fa57sueKZXA9+lqjf89+2e8fcC7xSRnxCRt4rI94rI+x6c8xexvn5X5cUGlSsYWt6qDMD/rKqDqn4/NtN8sNcdA882+z4LHB92TBF5Cvh64Mvv9gZFpAP+HvCtqvpLbZ2qXsEGzJdikkVb9+2qegkDo7+J6aAA/wp4uYj8URHpROTVmLq09uu9D6bW/fm7vdc7KF/rQPhzmNT35Zjk9LSqfrOqjqr6M1hH/JzmuO9R1R9X1aKqG+ydfKiIXFLVZ1T13553MVX9h6r6O37cd2Iz2auaXd6oqn9bVTPGI7wMeOKim3dA+jRMxX3G+8Tr/VrvUNXvUtWbqvocBoKf+Hwa6Zxy2M/w3yfn7PsEpm7/F9jA/ijglcD/6PU/CHyuiHyEA82fxySVdVP/FSJyIiIfAPxXTR0AqvoZfu1PB35YVcs59/Fq4P9W1euHFSKy9vv7lrt4xqf8nH8GA+3fBL7jYP/nsDF9V+XFBpVnOP/FtOUdqjo2v29iDQAGMJeaukvA9RZtnZH+YUz9OGyEWxYRCdjMuMOA40xxpP+bwLeJyOPn1P8qJuF8g/9+B8bjfDkGNJ+KqUdVWvs/MBA9fMEvRnmNql5R1SdV9fNV9W3YzP6xIvKu+sH05pc2x/3WwXk+G+vQbxSR14vI7zvvYiLyJ0TkZ5vzfjgmldTy5vpFVW/612MuLu8DvFNVnznnWmsR+Vsi8kaXEH8MuOKSwwsth/0M/33ehHjq//+6qv6uqr4d+BqsvVDVfwb8TxhwP+2f55jf/2v8HL+KGRq+o6mbigPqDwCfIiL/aVvnoPE5GFCfV/5zTNV9/V084ynw3ar6Uz6x/AXg94vI5Wb/E+BdF1zzwvJig8qvASIiTz7P43+BfXHrI2lUFFeTfhj4J6r61dxFcWnnm7CZ57NVdbjF7gGbTS56joRJIwCo6utV9WNU9SrwxzFO4ye9+pOAvyIibxaROuj+lYh83t3c/12U38KIwivN51hVv6TZpxWJ8Y71mZh4/Y+Bf3B4UhF5P+BvY2D8qEt1P89Z1e9u7/WqiJw3G/5ZTIL9WJcQP6HeynnPcJflF4CPOJCAP4IDdRjAAe9NB9c7bL+vV9UPVNUnMHBJWNugqu90wH+pqn4Y1rd+kovLXt/y8lkYaLzugmNeDXzbgapzu2d8w62eycuHsE9H3FF5UUFFVXfYLP18xdRvA75cRJ4UkZdjHetbAETkEmYy/nFV/YrDA8XKEiOLEZGliCyaXf4G1kh/WFVPD479QyLyShGJfp2vwaSuX/T6L6pSi3MTfw74keb4V7rqcwkj7n5LVX/Iqz8IA8eP8g+Y2f27z3mGLxCRp++sqS4s3wd8kIj8cb+nTkQ+RkQ+5LydRaQXkc8XkcsOtNcws+VhOcI63tv8uD/JATdwt0VVfxf4AYzzecTvtYLHCTabvktErmLSQFveAvyHF51bRF4rIq+7oPp1GNH7GvfJqFLrRVzSNwP/jYg87hPbl2HtXPvZh3v/e1/gG4H/s0pfIvL+IvKo961PA74Y+Ite93tE5NNEZOXP/scw8Hz9wfXPA436nE8Bf5CzUsztnvGbgc8SkY9ySuCrgH9ZJWoXDK4C//qCNrm43IrFvd2H803K/wnwAxfs/wdorAl6wCZjs9BfxlD5nf69+qm82q93g33b/Pt6/Su8vv087XXv5783B8d+vtd/DvBLvu1tmFn8I5p7/GasE9/w+/0rwLKp/w5MX30W+E7g8btps6buq4C/d4dt/zoOTKpN3Qf7M7wNM3f/KO6jgIH0X2z27TG9/xkMUH4K97M5fF8Yr/FOjKz+GqzzV7P2F2Cd8o6etdnnKjYg3uL38I90tqK8zt/Jr2C8lDKbX3+fb38G+NpzzvtNwFff4rqvxHw3ToF/C7yyqftKmj6McSrfgKkCbwa+tr5/jHN4g/eNNwP/G42ZHvhczLJ1E/OP+o+bug/ByNnn/Nw/RWNR832exPxPLuozf46LrUkXPqPXfwlmIXoG+F7gfZq6/w74mnPO+TS3sf7UAfu8ijPWW3+pX9Vs/3GMmf+ZCw9+WM4UEflhzAHpF9/d9/KeXkTkZ4FP0udhHXxvLy7h/xzwCar6Vt/2SZhqtwA+XVX/+YXHvxBQeVgeloflYTks9yygUEQ+VUR+Wcy2f4YDeVjee4pYzMh5n49/d9/bw/Lil3siqbjZ71cw9+Y3YbriH1XVf/eiX+xheVgelgeqpHt03lcBv6aqvwEgIn8f8+U4F1SOjlb6yJXWpP48gU7v9Mize1Vs1eaHtv8VBCUGIUi9llL5yOl/ezIvs1VPmyo/CQqI7SOCIKiA1G3NvU7n8f/mIyWEIFRLqyBICChKLoVSyt4j23V8bwHcCVXr+bTeUX1+RUTOfOoxMp1svx1BGceR7XbHMGbGcWQcM1rsHmIMiEAMgZgiIQjtBKcKwduklDLV1eeJMRJjoJRCKTCOI6pKCIEQAqXY9UUCCOSxtGSjPwdIsHOgikjY20drS9QG8TaZn7/2i723fbBB9+vupOj5P+q3fU/Qi45jfr9S+zFvV9WXcI/LvQKVJ9l3sHoT5sI9FRH5Ysy8xpXLJ7zmS/7LvRMcSlA2gA5bLZzZV1V9gFwML3oOaJRSKKqUbJ1qHLNtzxktBc0jsYycrBPHfYKs7HajDZY8oFinLVoYc6bkTMAGRAiBxWJhnV0LpQg5Z0AJYkCjgIRI6npUhBAifdeTukSMkVLKBAg2GoU8mg/hYrEgpoTmjIRIv1wRusRmHNnutgy7kbzZ2eAJAQlKCBBjIoQEJVJQhrIFApoLZcyUPABKSoG06OlSous6YuwMAEOwQQsTkNVBOebMc8/d4Hd+5y28453P8qbffjNve9sznN7YsupWHB0tWa0Si2XHo49e4eTSETmPDMNIjJGcM33fUQrshgEtynYc2ZxuCSKcnKw5OTlmu91x/fopb33LO8h55PjSmsVyxfXrG55913N0/YIxF97xtndxerp14BFW6xWqSt935Fw4PT2l73tyLgzDSMmFUpTi4CwEiioShJQS4zg4GIm9EoNXpl4qEGS/n4nIuf1yz5WkgE9XzYQ1w4n1E5tsDNBtP9GIYO/A+ol6/woIGRG4sRnfeOGgeBHLvQKV2xZV/UbMps9TTz5xnv297ne785z7/Vb7nwGhuk0VVetMgIFJKRQtKIUgULSQKYgWNCsl+yw3SR113AuaC1qUQmGz2bDoe1Lfk1JgGITRQSEEofj1yzh6xykMOVPGRLforSNJsAFXlDIqQSIhBHIuxAAQQGHYbunEILfve7rUsRVhu9lOAGrPPhCCddXi21VNmsjjAFoIwcAjxUiMdj2bjc9KL8Wfu5RCEGG56Hn06iOgsN1sGbY7TlZL+tSzWPQcHS8JUVgfLVmtlg7mIyKz1JSLEruAIKTtjmXfowqLZUe/6BCBUuDkZE0pmfXxir5fOgifEEPi+o2bpA4WJVFKIabI8dGKYdgRU6DvEzkPk/TUdUu22y05Z1SFUBQBQurpukQpoMUmBS0urSkUVdTb5lwJ8w6KiPUndRHIepW6cGob7HyRCi22RwKFGIUQCikpKUHXRRZpSYyB33jTGcfle1LuFaj8NuaCXctTvu2+FGv/s7PCeaBz1g7fgszZE1dphJJ9EBaboUqpyg9VoEAgBjHpQGQauCkllkvr+HkcHZgyIZgakHOGbJKSCuw2BaIgMRBLR4wJKYKKEmIgl+IzvIFKDKBjhiCmJiySdT1XCcasDmiKUIghogRyGe05cqHkERBiSJNKEfw5RGZ9wDQUIYQwATCYetJ1HZcvnTg4Z6IoqLDd7ei7RNcnQFkue1arBSJCzoVSMoiSx0zOZQbUFEADWpR+2bFc9vbMQH7kEqWMrNYLYuxQHTk6WlGKksuW46Ml40LZbrcsV0uuXD5mt9taO0hEdTWBmiqkJIQYEZd8gwROLl0iF+XmzS2li4xZyGMGrX0tWL+jTjLn9M0LpJW9fQKITxCT5AGgwdvaAAW1ficoMQ4sF5HlsmO57Dg57lmv7XuXEiDv8aDyU8AHish/gIHJH8EiY++43In6Ukf9ueBxB7NDq/4cAgsNqEhznYLr9d75bUY0FafOKIGABrUp1Geueu7dzjiGLnV27iCk2DFslWEYEOxcIcYJCNBCiAktsN1sCCHRhY7YJSgQQySXjEggiCJFKWNGoxBEiCIsFgu0KLvtDgZT3WxAFLtXgWEcQG0wq2YDESKBWRoJIXivr4ZDOTMTG4AWUkzQCcfHR4zjgJbMMI6cnp6yWC4m8F2tliwW5vycc6aUYGJ9lxjH0cEskmJ0jkDo+0RMwTiZYPcyjjsHp4DqihASu2Fgs+nRfMJ2m7kZ4eTkiEceOWG3W3D9+nUUISZhHEbGEba7Heujnq7rGHc7grffcr1mu9mZKhx6NpsdgxYMf2Xuc7cDjUYKb9tOVY2vQya1Mky0jqIqRElEEUJQA18d6ReJRy4nHrt6wtF6weWTNcfHSxZ9IqVwx1TOi1XuCaio6uhuwT+EJS76O6p6Jq7iXpZbgZI0g/zwmJmUhYKLor7NOJZCDkpQ28lEZiFE5xRg4mA1Zx8k1nnGcSSXguqI9nbtnLPxLTESSmEYBhhHQojGecRoM/UodMueEAPjMLIdCgtVAkJMiSjBLyoMux2h68hVjes6Yoz0fU8QoWDEadYRLQXIRuqOo3M+I6iSQk+KiRgCoSVpmwEUZAbdSmAXjGTNGA+0Wi3RckwQyCWzG4/oUjIpRITlYklKPSYlJru+D64ZVAJ9Z+qkhDCpKiUGumT3OIwdi0UPCClGJEa22x0lX2K9HHn22Rt0nXL58jGPXDlmu9kQQyYXU7M2mw1Cz27XsVguJ1Dpu8hyuURi4PpzNwlRuf7cTbREuhg5Pd2RVRizk+Kit5zTav87BONQ2xkhu/QTRFAxddvaIJIihJAJAU5OTnjiicd46WNHXD5eslotWC1WdNGk3gCo81/3q9wzTkUtrcH33+n+LXrfVjwUud1kcMdl73pqUkG9RgiBrAVpLCIh2OwvARe9BYlM+45VbVHTdiUEYpgHZHZrhapapx0HtrsNfWeAkbQ3IFJl2GyQIPR9j4TA9nRjKlBIEGE3bCehIaXO+B5VpBSkBOtIpaDjaOqDCNJ3pCHRpY48jJQyUtTAL4+D33smBDGCNpmVRSY+xTiElgxvGQQR68imtgS6EJ1iKBMgjWU0vsRVrJR6l/icXypp4hByzqDGH2lX1c9ZzURs39QlxqGj66z9YjCCu0sdKfnAL5n1uuPKlSusj1akpMQOSlZ2u4FVH6ZnPD4+mqStlCJdihQKfQcpQQqFk6M1165t2A0DFGF0ktXYpXkQn8cPngcsMUaCmDWv7AoxBFOvtYBk1ovIciGs1z2XTlZcurTiiSeucvXqFY77js6f12Rl518QNIR/P0DlbssLDBe48BznvcgLpRRwoha01JnYVRlRUomkoC6Gm8SCM/AF4z/MRGlcgJ0IUONRKs+ARMZxJKWIhIXNpmUkpg6CdwgzClNKYbfbQoCYImUsZEmsVmu62DkYGD8SumS8RjayN/U9QQJ5GFCF0CVSNAtOrhxMEbRkO6YYoCCFEBIxhomorW1cB91h+wUsek1EKDlbXs6YDPFUSSnR9wsD4xKtvUtytaZzbqM0pmX7H2NEFGrGg9JY5uz+AhIghB0pRrq4YBh3BhgxklIygAydSXBBODk5pusSMSjr9YLddsdul4CVS2RwfHJixO9qbTxYF9gNW1QHYoCTkxWbm8rpzTcTA4ylkFI1f1fye55MWgtQ2x/b3/VeS8mmxgJIIQal6xOPP7rm6pUTrjxymUcuX+LSpTXHRyu6LhGKkdnWrsXb3a9bivNg96c8MKBSy7sjbKBaa2bS0S0ZWs13AkVIIROCzSCSbLCoFJt9s4IEJJoZDw0Uspl5BbckGMcSU28Eakp0Cx/kuZALhE7cmiMslolxMJWp5OzikpJ1YKunHB0fITGAFLa7kQ4DryJCziOxJHDrU1GlCzZrdSkxdom0S5BN9UFrJ7ZPSmZpcnHAADLMPIo7sjiZXM2qkEuZZlwNgUJAihp4dJ3ZKaKBiijEkGauBkWCS3kSJiuVqBGgAhMxWrTeYyBEe2djKEQxUlIEYhcn9SmGLVe5TIyR1WoJAstlj2phc7olj2bKVi10XcdqZRxP1y1MigqFfhCCFI6OVuRRefOb30W/SK4GR/qUGPNIGQdUrc+YtDr72swU9wwwIiZpLhYLkEweC4FAHjOrZc+lowVPvPQq7/PkFa5cXnO0WpsbQRBEBR0zYzGCuZRs13ArHCiJwKGqdS/LAwcqF5dDMnDeNkkZB0eoExyzmGmdTWtkfyViRQhOAEq088VQiTQ1E7OAarABSzQJRjIF03kJ7iSGECNojKZ6+IuNIdr1ESc53ZcgJfpuiSrkYSQXZbfdGgksdr2YOko2AjcPAzmPCHDjJqxWK2LoEGC3HQAhJEHHwm7YEVIixGQ+F6PxJwEhiRGwIUSyFCd5DXxCmJ3TxLkakdmiUd0g1H+IWE314wgx+jFCFNtOgEgg54wgkxqZUnTOydSkEMV9hTACXIJrpcUvPKJkxCWoGO1/13XmF6JQrTAhJoJCsmQYpJQmiVHckldVpXpPw7BjuVxMznWqYh0pVJ+PyGJROD3dsFot6Zc9MZqJO3WJYYDtVhkHl7BCadS12SYUmh6dYuBovbR2U8hkimb6ZeJlL7vKSx9/lCdf/jiXLy2J0aTnoBi/lws6+sQw9fuqItrVxraz34fywILK+RLLHaKtiFkJzngzmlnXRESd/FGqsUcQH4AFKKjKNJCo34OiYtaI2EUzk4bCmMdJ/GQi7AIhBuctZs/U5Ca+ECODmEoRY4BeGYeBLkWGPBr4uamWonRd79fYkkuGYcc2BmLJdIslZRgZx4EUk3WqPJKHkRAMVIZhIPl96NQJrV1iTFAGhECI5unaOttpUYpkgpoZWIuaaVNsAOecDUSSSTf7BjgnLoOg2cTzOA1uIUahSJktTNH2EzH1RlXJ3idyKahUrsO8ca1EYhTyWOZu4lxCJBEWAbRzaWS25okEYojuhVuc0DbLnIGKTpNS7E1d64r5pxydHINA33es1gvW6yWb7Y6bN7bcvFHIrnKIZopPcEJAUCNeUbousT5as1jYUEwSUe3oTnre56mX8fKXPcpjVy+zWvXG97mxYDcama65GDelTG1lKo/Wru2A814HKueTs+fpoRef4uzxh2RYFUEnMKnXKM7RqvtouD/J5NCmRiC6E70Tj2pkYCcQlB6zXmT3Rs3ZLCwKZDV/hqKDq0H2vF3fo0FYHx2havq/RFgtlqwQhmqRGTO6slk7DR39csF2s8H8KgqFTIijqQDDYOStivvBDMRkM3Eei4vyMnnoTvq+q31INIe4BlTEfS9o2lMxiaZt2945HJWW5zr7Tg1E4+T30r4vMVESvL1bDmcS52Oya1Xi2NWnalKP0fkX1MzT4BpbmMCjvSbBuKNxVFLqpuNbUKllsVhMvjRdOiVK4fLJmktXLrFeLzndbOhSQNiy3WQKilFegkp0kj4TROi6xGKVODrq2O02PPLIZSQPPHLlEV728id4yaNXWK86ughonqTNUjIUdf4sO+fUjJHJ7uCe4q7e3q/ygIDKCy9312RCzS1s5uJG562DMWeTBqbBZ7bx0ClFEqjVDxmUbJ0xVa/TSKe4Kz6EECeflmHcudg6MuSR7TAQNxHtF4xjNmkiRZbLFaoYeYvxNzFGUrdgHHakfsE4DuYVOrm1J7NknG7put6ki1yMlO16tGTzwCwTdUKNMdJSCNGkjpjMT0Im71kmcLHm0wl0RAxQuq7bA6Hatu2gnGZ8986FfeDfC0NoyNrWazd1nXMQaY88nveBUKwua3FSVxxw5vc8q8NV1ashFS2QzYBVQdiuVei6HkrmaL3iaLXm8pXLLJaJ7W5B30W0XEPChpwNRLVLoJHNdmfSThfousilS2u07HjJY5d45JETjpcdT77sCY6P13TRQDJosb5YPbhLMae8Yo6X1Xdn6s+NX5UWrR779608cKByaK25m/3PVrb/Z6mn9lnzHq0WnzKBuSG8HWKTmnEjGaUTKNEd3OoLEzNJBrMpmhgdwnSdGJUQI4tlx0IWE99ZVNkOxd3ag8/iZmYexjL5cQQ/V1EbAEF7lEK0B6Jkc20vORNDMAkpZ4TgliHjHfI4EpINoFkkLqQYyINbWgKkzolRn8FN5YK6AkblTur7aaWO1rRc3PolzomI+49MHqGNpDQBUXF1w51eqoqCKCEKvXTG2zQm0ir91L5gIBEQDcToOdbVvJ9tWlcHP5PoghFoe4BSz1etURUMqztDCObe//hLHkUksD5a0S8CwziSYmC3G+i6wG4YTCUjsdkMpvL0iRCVk+MVy0Xk6OiIp558KUfHS66enLDqO+98RvQXsFis0a2JZfadCq7mZweZ1qfK1ObiVMB7nUn57pnpFkzamdCsEmIduVo08InPCSyduz41CA5m9Qhwd3mPaSkVZApjViwWUNDROdcAEuJE45Q8kkcnycZMHdR9nxAnQWMMdP2CFDs0GOkWY6BLEe2VXDIxlknayZX0Db1ZfFQInRPImHelikkhKSWTXLrkKtJAygnjiWbJSXX2DbHARQOREN2RrhLbYsdICEgMtWbiTPbMzE7uVkdBXK0Ub1NIMwDJ2fc3SyZVsqmCixHV0Z+p7n9olq2cDA40fd+xGyAPFuTpU3uzrzucqZC9rpWM6r5m7g3efvbMq9WCx1/yCDkLi0UiLcx6JGoc02a7Y7vdoUXII7zznddYLxbEBF0fODlec+nSmidf/hIuXzpmtepJCMGjrktWiy9rrX9Nv5bJ9YCJb6HlVNyUrMU+96s8IKDSgELdcpcSy96pdE4xcFaSqUjexmDM55/FdZ317TqMSjAVoihaspOCggRBGSf2vmihYM5tKQVKMSliuzUyt8bdLJZLYly6hBAYVRAxpCpaCBG6rpusFNtxpGy3FtSWR1arFWOwWbaIEGPHbtwRYyJnc7dHhFEzKWckBetkPmuXkn0mZnZyC+KWEzN5q1j8UqyevSIuZajFHjVkapVUJn+gyoAzqyizhOHeokVNlPf3Bm6ijvsrcbTq0nke0YdqVt03xkjMmd1ovjjiznhBjDKNNUgzj2a3eTkAACAASURBVGc4GoBSlBC0uf9ZOluv1/T9kmGXSV00fg0YhwIS2Q1bht2AAqc3tuRxR9ct0DJwcumY4+MVL338Ma5cPqaLQhDnTLIRsDoqOVepJAJlAnKj9UoDMAYvFYydUWpU0DsYOy9SeUBA5Wy5W7t6FacnaQXO1yMdxFt9//BT+Q+d9B+l0rT4iwshUFxZFcUHSLaQedeBx92WGBMxdXS9haZX0b2Uws3rN4GbIIHFcslyuXY3fshaCClM5GeMgeVyyTAME/m23W7pe/dGzdF8ORWC8y+73Y7UdZRi1qm+W1Cnu65L0+wPTByTOaxBSMlABR9Q/pkAZPp/IGnAbAVjbueW95gAqMxEYgsKwS00+5LL2UC8ei81vUTbd4qnq6i8ibgEFRtwq6ATokmjc4oJk9RMdRUP9JzVq/r+l8sFu91IjEbuBh9N63VgsVyzG7YmteaRd4VrdOmqBWhK4ZFHLnNyfMSl42OftBQdXN0pnnqiqqjnSeXWAD5JWJcPWgMMmayNFdyLnLdAwr0pDySonNeB2nJLHuXQknxwXFV9LgYV6+Q1Rqea5oJEErNILAjRZ/nKZeKzRp09cs5ujTFwSbEzh7fUkfqOZbdgu73Bbhh47tozjOOOo/URSEDzSB6FwSWearVBzP19ETsGT0yU3YNSgdj1ZvGJ0SSnap3ZCz8w4jKlyDjmeXC5A1rX9SDuQ6JKTBZHM5GWLVCEQJDgKmK2GCgf6O1gaGf/82bPQymjtdAcbqug3EoWM1jNapG/Gn++5CrYPqgEH5jiybdUZ5641rfXmczQMVLUne9CIkSpzsMsl5259I+mDu12W9BCPlkDwtFywdF6RZ8Skcp9qLkkuxRcNJtq7qBbieW2Pf2LR4E3bc3+GArBddj7VB5IULklaNzmuOnYhpydziZV9dw/v2LpDOzTsrtuSlXIatxIUlCMWwhqJy1azc8eFUxhzBDFXMjHbD4FJWfCENiFwGKxJKZIt1ywPDriSJVhO/jgV7oukpF5Ji+W52V0F3h19WTMI6nruH79huVO6TuGYibtvdnbOZCJz1AIsUOKkcgexUTWQuxqEqZIErFBE9y/Q4UYArlkutRPfEdNolVVx2nlztr3nUuyZFjGD0ychXMZVcq0QeUWJnzQ2At21dajq1UnM/FBT5itPQ40ll2OSX2beJNqupZE8fQTJrFUCWk+d+sHYvjm/j3NoomKWvQ0Qgz9ZKFJ3uZ919FFl5xKMerDIhqhKLmMs/d1w//g94wDXjs57kl6pn9OdWjd870cVO6m7ImD7fbqNStz8KF3931VR13NEaYOpojlK9HZ7IhApjACpQqsbj5NsTNPVJgkiiBmbVHMZX3MGTDgCkTGYQuyIJeIlMKyXxBX/TQbp65Divm0DMNA3/eoKklhs91a4qQQTAVaLKaZUoIQusT2dENKdu1+aVnnAtElK3PK05xJyXxlYnSXdnzmTXEaeObkFl1dipMUYHlYLEap2i3VAcMGtquHuId/tbz4USEYB0Uxq4xqoeSCVpDIxfml+ioth4z54IyICjFa5PU0UTQSkWFWsUx3SVC1wZ9z9vQBYQrDkGL35RTVJKFU4Ku/6yA272n36ZE5V46JOFVltsj1KImYhL7vDJDzAOoEfB7RbCbi7FkGzdEyuPRhkhQVUGgAxfu/pR8N3uuNo8IjxFVaR8f7Ux4oUGn9G16Mc11E5B2WFu2n44N4aj8bQEEMKGLYD2uvYniM0UhHagew8GF1Hd3yX5jfRM5bUkqsCcRFZNju2LjHa86ZkxNbfjhEN1s7RzC5mIc5R0ZKiY2nQmz9LFKXyONIWiw8hmcGQYuyDgRXYUYPeJykG2FvENX9W6vINIOfQ5BO5GEzs7ez/EWkam3P6bfpktSHrTNxKZXYxTycfR+RWQKZ+gCzGjNd09+ZhMn8ZO0d9rmb2fokZ0DFzj/ft6h5YSNm/SulEtEWxxVTR991ngTKQKQ4f4Jbd+q7rir4QSc1615TanzVZO0BYlXRauzRBXzUvSwPFKjU8rwa4aBzTnr1LfiVw9+zx63NOHvel6rusFYYIyz6NBGApShRTC0Qdce6aH4mVa0ynb6AZy8TAqc3T1mLmZFLHum7yEZHttsNqbP8scUTQLV8Qt91jMW4kMVi4RHPic1mA3hS6BAInbmb930/WU+yq3hR1cMFoIyDAZH/rpS0AaVzF2EmqlXVCUyxiAQ95DVkb3DXe29J2fpMles5FONbRztczbRzzOAicT9RlIi57VeP4RACocgUCDmdO85mbSOVy0Tkishkxm/HcG0fS3dZJTixwD9XxaqaVtWXmBKLLpJislwyw2hpKLRY+odczb2zFWfmQWaHvKlPTzTKPreiquaGojr5+agnumqd9+5XeUBA5ay+d2j9uRPi9m7BqO3Eh8dVx61D86S4t23ROM0KMQQ8QSsxpiazeyQlQdWkliCgg5mTx9H8UrabjSVhzoUCHB8dMYxmMbAgxNnvZBhHjo7WjDm7AmZluVyScyalxDCOrFdrkzpiZBjMpNn1vZmys/u7TK7okd4HXV/3EZmet/jgUdknS0UCRbP7u7Qzu5z5PwFIKw1WYPBZVseMjtnSbx4UcaLxULppid8aP1WJ4tp/QtVnYHqfWlXaSuc0ADhds1GjWimltksFAdQc06pgJdgEE2KynMQxIqgFi46D8U+lmHesWjpRJ2gmSaWqz/XZ6+1XAD7TPvXe1S09pbg3dDXXn00IdS/LAwIqF5fzJIrztl+0r/mjtL9ny8MsyUjzfe6A0yBQY2hitNSMRYurCm4WVRvApZgIHaNlgR9z9rQAc9Joy1NiUsWw27mE0qEiDJsNu2FHv7DBHWLHmPPEqYjg3AzTDFZd5MHiUjabDYqyXK/Y7XbEFI0ods4oqAfoeQxSTMkjhcskzUwWJZjNydX/wRrOzJZTGML5OVYmiSLIJOpP1v4yD8pc61zFNP+XymW5tQqPVm64kwomtT2iq4uVN5skJlxVcNI9uKo65QPmrOrdDsIWUGpdTTFRcnafF0yak5okPHjeHZMexmE3SyOlkr06gxOm7laVrTSTRuVQWhXu3H4vYo6Y6hMBobnme52kcnvp4lYSSH0Rh7pO6xHZVu1Nlg3IVL+AQ53fdFY3+0mmC8GD1dxTMYiTnub7ITEQUiBqRIrNZLvdzkXonuVySd/3XL9+nWHYcOPmTbq+Q0Lg2rVrrI+OWK/X5DCa230FAWCz2bBer61T++xf+RZwqcUBouu6Pb8Wuz9lGMfJ5K1aiCFNA7TvewOuhkuYrAoeliBYGoPJTB3mKOU9XgqMAFWdyEiRSoqyty5RO3AdQnzmD0SCe/nC4O0ge/yHpU6g8SeYASWAWv5edZUxhnkpCwuFaEDFjy9q+WJjCFP3PFTxoucRbjuWAbMg4g5sxWKv6qoLdQ0PdY/XIIcqYziXUzmUNm7FD+pE7Ipb8cJ7I6icLYeNdkig3a7M++9nezvv06pBVZw/nLHUScNcMtvdyLILLPol4B6QkhkGAxRRMU9YlN02exKhxDgWNptTVqsVXddxcnLMZmuxOrGLxNRxHI4ZdjsLGrREIGZ5Kpl+sWC7tWU2UkqWz7Zpj7bjqVpIQrdYkD03bmx9TSb+yAAjNuqQ4tHWjTRUgXu6VmMCPvQpmdtPUNyi07R13T83kk6rIrXxNjVsoKocUYuFRCjUbGZSr1MysVqndL6/SqIzgRYT7zKDkFtTSkGpOV+SP7funXNSgUp1rototoXLutShpVgO4HGcMvOJm26ym44nk3sDKNa+M8i27dm+3xbcDuv9lGfGwXu1+nORunPR71udp0og+9sO689aJNoy6+bRraEBVVtErKSMFObkwiIUX1JiN450Hhi2Pb3pGcgSUnCpRegXHSUsCGNkGEeiKH2XiAF2uy0SOo6OjlAte4N8u9ux8ND/loTbs3q45FEJ15wzuQEOqFnUTF2QiQgtPkh14mmqU9wkrsv+e2hVkFqK6hRIeAgobVsf8hWV/K65VA5JSRFLsJ1zjWOy3DZT2AFxGlWKJTSy8VxA/Vp5Xk4lzGSIH3cwcH2f9n5b0DSpp6CeAErUFj8rrl7WQFUt9pli0nS/DQ/Vr0OVsiVs22MmEvzgXBbjtn/u+1UeOFCp5U7A5Dx4OXsce9JIq95chPLeX2YmXbEQdJmTD02Do6gt1xEtcVA13xbNjKOQ+g6RNac3b3J6umEcRvrOcq8sWBBSsjSEN28asIRA3y8IIZnEsuhJi97cGiYJZbQM8nX2dtCoA7SqOfW5ohO2xcXtooWus1c/5pFUuinWpg3xryAgTn6I2McAwMlFZPLmrCL+tHyqg0rt/IcifU05MEsAFSSY8882KmwIlqGvqE4Eq4GnziRsqT4wYUoijYPDlMDcpRJTV+oqCPumbxHxc+UpillVCSlNzxFcvSqi9H0Pakm2jCOa8/7avVlEvGW+0yl1wWH4gj3WPsBMUlvjiFeq859UXypvw/1ZlGoEOesgeO/K/bvSHZbboeqdo65wNqP5von5PFCZ6koLOj7NYOH3pp/XWd1n4NESKokqUWzJUgmRISt0HevLl1mdnJA6S1Zdcma72TKOha5fcHLpMovlkmHIQLDs+MPIc+96F1IySexaKSS6mCYOozXRnnHOatqqJjWqVgHFQvBNtTKwocznSSG6+72ZoKskNvMs3qaqloM2F1sAzT/Td9XJGtbeW6jreTgtKVJBRSZfm33Jpb5VIdZUmApRIjV9pGLqaX3fVd2rgohJOcqUYCRU/mHuI22bFYr76zVBeznjgelT7zJoEfIwOnibBChqGd5s1ceCFktRgTi5H2rkdpk+FWjbPrp3byrOyQha6pXDLGlO/VpQCZNvy/2UVh44ULmdejPr9HdzzpacPavqnAGW5rf6zDuFn1dnNK36v+dy9dwX282GzeYUzcWSLS0W1MjWK1eu8Mijj7JYLSkoIVku18H9RBaLJSFFtsPOzNFd4nS74caN6wCT1FCTQbWcx5ya0TxjK+jU+uqgVzufrQVcrVYz4dtaQyqBaV3XHPktuC66FTR4B7cASlN1ymTpqedrpR+bXUOTm2R/0FQJpSZyqt9r3txDtUCaY4Ofs9ZXVW4OUWhB4yzPMN9DnJ67ckaHatueGVfNh2m32/m+rRnXosVreMJkypaq+smBtLLPH+6pgFqfbe63Oe87bdb7aVU4DsDyXpcHVv25XZmF4otqz/ImrXObngEO7ywNaNmLKE6mzVamXBTU/EYWC8v4Jrn6GlhE8M1S6DwuJ3Ud6m7oq6M1XZ84PT31hb/FeBkfcJevXOF0s2EsmdT19P2Cdz17jX6xJPX1ddkyFXUJCnDXc1eHDkP46+BOySOQJZB15HSzYX18bObhYv42bdyZNueo7dp24Ck1pkszxQHFfG5s5BRlb6DXhNL2e+aC6nWquXaOUWrTDuzzL1C5m9kfphpoK5BqU4fqHmhOak4zCOt91j7S8ksTMDtnUddzKqUwbHfTuWvjlWLL0YK1ja2kaMZiW5qVKRGX3Uee2qTlUOr/rHmaFA5VplJmngipe+GcmTAb8+99eeAklYvK3SJt5VLm32fVnDOzzuF38FnOZ4tSpohQxRIwFS3msCnV0mALaGkeGbYbdptTNjdvIkCK1ZogxK6j63ti5S6yeWBKCCzXK5arFSEGixhWePbaNe+0dbA0WeYaaaPt+BVoWrUoJUsU1XWd8w4BCxac9fuJ+NWZO9oHF0DEvXPLZBqu4DXm8QxwtyqZtJ0e9gCjckJtaoJ2sNdt7fZqvWpn9cP/IOde77zvLYdxxmnP2xx3jqy5iHMep+ecpZfqGOjSXsvXeNtWSa6uZHne5DeR3I0ESNN3J07qQNJ2KuxMv77X5YGSVG6n+tzJsY1Uurf9PJXnkAiT5rtZTgqlWAeYFx93C4HaUgq5ZFLfEUqZlpZQLEJ5HK3DxdRTstItRpbrFd3CgvhsTZ6BzXbLMAyEZJHHFIsX6k5OLJFTl9huT81qFAN915OSxZHUwVVdyEOwdZhbotXaZZ7ZTEUq1HVukptBwfwyphUHqopQO3jb3tJYeCbrlHFFk1SkNR1jIXhC7VLqintnPVaBM2pRBcz6nupzzY5gMvmsiAQi1fNUPPtZldx0krzqOVvLWSv9TCb35hhllh5M5TK/l2EYpkXd5+NM8ivF9pv8iKpkh6mJLRc2me0bLq8FDHs/wdrOLrb3TquFSh3sJigV8Xf0/MbV8ykPFKi05ZABPyy3aqOzFqByBkQuUn/qtU1Ct8GZ3QdBQiRnBxWErMaHrJYLNCVUM2U0VClabDU+MRPmkLeomrm5W/RuLYCRQBizreW73dEjlkoyRIJapz4+OXErkkW15hjo+uXEsZRibuIxBna7LYvlaho8qpaPZbfbAZamsOsSJbi6RB3omOfucjHNtFXslxiJEiYvz9pS1UO2+KxcJl5FXeyeQlLsd8H9NdSXg515odruLRgeJsduP7PKFG0ZEzVnNXNr3487mtVBGh+Y2k9mC0n7/qeBbmLoJB1UCQZh4q6m8+h8zTamqd5/zqNbbeoz7/e5WeLApWMDJ9yahVbX/TCD3bnjQydJXVX31aH7UF4QqIjI08BzWHqZUVU/WkSuAt8JvAJ4GvhcVX3mhd3mnRcTKw8Bg+n3oaPW/rGV5W9mLRGiJPPwlEgIS8o4kMcdlMFd4JXYdeTRQ+dzFW0HEKHrlyiw2+6M41clRJMyel+c/XSzoWhhtxuMo5HAUHYo5gFr5KgtFjaMA12/oO/NAa7r+klaqWL3oSWjVW1KMUAxq09gSmnQmB8tBsjVn+oliswAjVkiaoDipH5VgGAfvPfCALAUlTSSUyvqH+aCUd33cWl5kGBI0bzL85zDoKZhaD/OYXr92YDIClR73UTtT1V5KqhU7mWyxkmYIqhLcS5HdM9vp95v3cfaXtDSmIClTM+kzfOdAVjnTaoHcF2Dug4K4TzwuTflxeBU/qCqfpSqfrT//grgR1T1A4Ef8d+3LXer+pyHvK0K1CL1IXt/npQybVdfPmMYzDzYvJzqxr5arTk+Pub4+JjUdZ4oKRFSQkJCPIVkDGb63WxOAUgxMu52nF6/YZai7ZbdOBB7W8unc1+H3WZrMSXBJAlbDKwgvgZzKeaZW0Xn3W43AUbf91N7VJ6hqkWzqM/0PNNs7y06z7zWyqrqalye/FGmLGOebU5VpwXcTcq7YD2f5rs0pG/rWduqBIe+FeeprG6gOXP+9tqqZQ8ZtHmfvmXv2Pb6VbWa9nXQG3fDFPtUy8S9TJJGVe/EpY7KeyWCJGLo6FIPGkw9lAi6nyGvBbjz+J1DS1GQ2adGxMzuUj3v7lO5F0TtZwLf6t+/FfjPXqwTt53lPCnjInfx2wJJPaYYoFTiLOfMsNt5VLIyDCM3b95ks9nYkhwxsD5aE51YTCmZrhsiMfhayU60bk5PKTlPbty7042BTLG4oEo+1rigChRLV2VUTfWp6xRvNtu9GX0YhomkrQGGwBQjBEz1FTjaLHcxmkg/1ZU5DmpyN69E4SRWz21dzbWtReq8Mr2Hg3e675dydmC176nliWqW/xDs/lsrUgtKh9xS+/0QiNpiuWtmVTyXYp7IB6U4iS8iU54Wkz6cj1Fr177vSV1nuYtrtHN7r34b+QCw2naZTN7NfVXnvfp+qvOf9fHC/Yz9eaGgosAPi8i/EZEv9m1PqOrv+vc3A0+cd6CIfLGI/LSI/PSNG6d3drG9DumKyhSFXN21Zwe3uTM2yI6475PprsF1/JrkpqhQ1M6bizKUwm4YLY1jELpo+TGiRLanO65du8HNmxs0FxarJWnRQSeMwdZiCcHiQQJifgwOBMM48Nyz18g3Tok5IzkTxSKh+0VHctVmu92QktAvLA+qidG2VOhut/UBHSdpou2su3Fg9PSQM+1Y43vCJHKXUlM96N6sGKSqgiABso5YZrdsjlwlE9Vys0SYVB+qSnLw/g6d8vxU0wxNMX+YWFNX+jsvqmR0sjbNM7Plzq3WouA5fBFzprN8vkpd/AxqPE+cpIjzAGVPUuFgjSGUImrLqohYHBKBrJAloCGinjg7oKQQKMMIKkhMEJNlY5OzhGwQIQZbEjfGNN9rSCg+WTlgT7lvJhBz3sbVUQWywEhhJKPx/qk/L5So/ThV/W0ReRz4pyLyS22lqqrUqK+DoqrfCHwjwFNPPn7HTzztKPWPWwCaWbNVfQ5nuMoJ1BPV7/N/39fPg2I5X0smD4r0/ZQnI6YFpYyMw5Zrzz3H+ujY0jx2EZGeUTAvW8kO37Y8R52pht2O565dQ1HWx0dTqgSz8HSMWIzRMOxYLBaIwHZrANz3tvzmYhHI2TiRYRhZLmdRWEJwzsWuNw6jdfSUzA8mma9ESj4bi4XyV77APDyjealWIKjkpbqbuTC5rqcDbmRq3/rKWgnEncTqwEWZB4enk6jPMfEQUpkdn4Hr/TSSx3wxJo7IJK59s7SpfL7ofW78kJqBPnWzem6ZHQlhjgmytpqlQos0zrZg+3Y33X/okzkbZmvD4gmfKuE939/so1N5p5Yzq/fR8imVVIdKVvvEUMNGylnp6l6VFySpqOpv+/+3At8NvAp4i4i8DMD/v/WF3uRULhBRm/vZ+8z2/lk10oN9Wx21WlP2xHCPkA2uHgzZMrON44CI+Xsslssp/H/Mozk8iZC6ntT3RB/Uxk9Y9GrJmdVq5Umrn+P69es++OPevaiWiUeJIdF3iznXLEwpFSo5e+rZ38AXWWceCMkBopKmLbFb22SW7oTNZuMSDJaU23mVMmYP659J8FaNmV/XvtrSvhu7pqlNLXEKvkRKLvtkr0f37t1nc95DVeasWiNunZFGSpljiw4tUNNxYibgmv/3kOSdJcdWJbHnKk6YVkCoS7VWoGwlssN7blX5EMLkVzST7ftBpNWiVNu/9WI20H4PiP0RkSMROanfgU8Bfh74J8CrfbdXA9/zfM5/KGXc7XF3cvx59YeEWOtsJDL7bAzjzlIPqFkv+q5v9NjsS5eOjLmgQViuVyyWC7fmDORhZBxG1usVi37B5vSUd7z97bNznRbGPLBcLei6bhrgq9UxqjOZaAuT4ev4GGjVbdvtliDVs3Mwx7xkfEsFlpZjadsk5zItMZLH0QZ6YzKecq8ecCHAwf+zfMg8IOA8QJmsV8wDaPZwZTq3sD8QW8LykKxlkjxnEDFwqVzFPjC04FIBJMgcm9RKxlMkdANuISRLcyCWQ6frF0RJgKdhaEA4OG/TPkd9l61kNLdF40Hc9PN5mRX2uC3rL/uLs93L8kLUnyeA7/aXl4BvV9UfFJGfAv6BiHwh8Ebgc1/4bZ5fDgfCRbNii/6qap6x5wBKOwjal2sd0h2T6mylMIw70JESE8JAlyoJFzzBtUX15pLJqnR9T69zwuM8juyA2CVClzjdbHjrW97C1cceMykiZoZxR/Q1fqopuetnr9jT09MpR4upQabCxJQ8ilZsrWQnnztPvmym8H2P1f2BaEmURyeAQ4xoVktIVUyc7qU3cR6ZgKl1pXdJ3f+3UqZaljP2wwmgtZCEmRx2EjJImAZwS8y276p172/5IRELkBQxzmyWTupqAftS8GHfAXz511laaF3sUZ0JWlXGMvoyIw5U1amwNJ6yfq1DD9nJvBwTXZfcP8o8bqv3cJW5h2Gw+wyzl3XlmGar2v0DFHgBoKKqvwF85Dnb3wF80gu5qVtc87Z1tzMdU3Xxg3Md7ltL8LV7TPupDlYmbtqSCuKRqwYSC10SY0eK9pJzUYiBkguj+4fYmsjW0QZfriF1iUuXL3Ht2jWeffZZHn3sUVshcByRaAmuc8m+0mDnQYgmadSVCkMIvkh7suEaI2PORGy9mbryYgiBJGnKkF8TM42eTCgEW8mvgoVAXfCEoD5oS2G33bqKeL4TmxamlfImbULcU5c0Dch2xm2BYc/adI7ZV2KanMDqu2sBZXqHnuN2sgrVJVb0LCjV6x+qF7bPWeCd1YuGWK4cSgwUZu/fmrzJvF4LIdq6TXVpDrs/W2kyhOiAYmCEA1PNw2KpMC2uKESZAGWKTTpwIHw+Uv/zLQ+sR+2dFJnf5V45D0wOZ4e2fgais34AcwcVj5MBEc8x4pndjQAdEBkIZHLokNCRUu9rJGfobJbNZST1vTlOlWyzn1hyp9PTUy5fvsyNmzctIfbRmlLMT6TrOspgjmvDyESydt2C4p11uVy62GwRyEfHR0g03kWCSQ6DL+8pnl9XZVYFzPRZiFJF6MAw7CzQsMwxLZPqNdgAKQezvB0byZqJ1e9iWnZTLVjf9Yk2nWIdLLYeUMOJueUNnQFrkogcFNpUELMHbc1B2/IolXBmT9JpJSRgD9iqilS3VxUkVO9dZDLZVqkjxEBuuB9V9Zgo83vq+45xN07LstScvOpE+hQoWsosqWGqV2lcH8ziZfxQtfodRqnfT0CB9yBQOZ//OB9A2v0PJZcKHhdJKoezlmDh5aggUqZVB0Usr4fN0EpEGYctgxb6TtFo0cwhJWJncTCiIF2aLECjcxYKtth6MYe49WpFTIlSDDR2u+uoKsvlwvXsjEg/DcbVajW54YPNzMOw4/TU1v+pC5EBk54usVp85sxw7YxfAcSyt7lHpojFKxUzi4ta9jjxdq5idwUeDeZpagvYV3BuXdQLSKQaCG3GBdVsi5qJ+CIFc1BnKzlUtaL9faj2NG8Y2M/ahurE21i72bmq/09V4SycQG2ZFRHGsUofdl6bcxqLTAhTCs22P5XsPFZIwJxnpvXRqdKnquJGsQlMzepVJ1IPJdB52do6ubQk7fPlJl9IeY8BlcNyCCj1/3mNWH9OqN2ak29xTvteSUQrto4PBOqyEJFQMpIHosC4PTU1aCWELlrG/GFgsVx6+LvlSVFV+mju9aenp4QgrI6OULVFxcZciArr9Yr1kL7QuQAAIABJREFUes2NGzfIObNcLhiGsSE8bdarwCFOzPa9Lbk57gZil6Y8rZOurYXUzaStqQhhmmmtY+fGTGpSRSvJSTCuJpR99/bayaMkH1yzq3n9bypQ5TYC+2Bg6wpFV+dE7LtxRAd+JA0gHhLGrRq0r+bUtykw3cP8zuugrjyO8WwjQWbvZBFhGIYDdahew3OkB1tOVnPxHMGWJgPNbLcbxKWSCuIVCCZwigdAWvAkVLOUVIn3CiYwTxKHYHm/ygMJKhdJEQd7ed15Ugp7/+d9K8zPgNHuUzuc+hGmLtlypkbKFQJi2bQiDONIitF8Vro1wpacb5LHHafXr9EtMqlfoATG3db14zKx9Cn1dF1vvEsQxsE4l7rGzTgObLcWGrBcLhl94fWu61GVyfdgH2B6xjFP/ilVxE8pMZCNbBWZggArAKDm1xHE4nlMCoioZCy/6znqJKPluo9CXbZU1Rb0ssECiKWIgKo61hyy1V0fjySOzpsIwWd+Wv6kusxXy45LWXUN53ZJj/kNWnEsm/yUbFsDBq6+aDEgiDEyDhZnFZMFlFpksR0To/kGKY0fTe1FPpBTBaZiKxMQxFJigGX3U4ixp+bWjTHRdR253qPU6O5Z2silAWXx/DdqPJuIO8Qplmjb03NIqG1y/8oDCSp3XrzXQqPSWKeq+FE/rWm4Asp8jn21qKDmEuEd1GbV+VqlFLIInc8Au2EkdZF+tQZgOD0l58KwucEw7uhXR77IleUpJZi3bimwWh5x9eqjPPvss4yDZWfvus48R4MwjgMhiK/7I+x21tlXqxXjOLvnz2y/TIuC1UFaivE1i+WKmAIyzpJOGxc08xp+LgkQAnmAFMwqUQlqI3ddDSh5T8URCW4lKmgwwETVHPuEOXaoNCI+7VpD0ZM+OfhEmdbEmYoDhWpxAl2m1JDnc2JtH9lXdaxYAiXz7nUS16YQaprHPAyOdba+cYh2n7boWmHc7kwanCSimceLyZzQskuZISaPRrZJIsbosUTqzySmdjUSBxO5bu1l5vf9NBKaM5E6IVgf1nB/eZX3LFDRmWidG2mWMOp2PQMczSnO7MMF3+vLpWKJd6hqUp6lmigmpu/ySBcT3dExpzdP2Q4DjNmAoLqJ+2qD/WLBOBaGYQvSc/nyZW7evDmz9imiwe6nXc60ltPT04mYrYCw3W5dRVp6MiaTjOrSqDlnYpoJvVqGYdjLINf+jzGyU0vobGvvtMmHTPUyErERsRspyNrNSU5pGrJ5h625v90GbrkJM0m63x0mvXbynq3Wpn0pVVwN2z/28PrV1b/6+dR8M43GTF22o6pkNQ4q+3IpXdcZT+ZgUyWZ4n4/qraAW03ONTsoWnzZ6Ksd1lUp90Cx9kmFXEYEj9nCHBLzmC1H8uSsKdbv7q+g8p4FKi2gVKmkZsva376/3yH3snfOZp+q7ijz+Q1IXP/32bEFllm0lEnPDqXQL5eEvmc3mIWnel4O22zRyAjL5ZLT7ZYyKIvFgn7VT7rwqJmg5klZnddUbb/qv7Hb7SYv3Oweujdu3JgGX9d1e5xIVktOnVKaPDRbJ79p6Q6XRGbi0XxTDCjsWa19yiQFhhB8XncuQtkja7X4+hR2+PQyZ+vLPphMdZWIbaKFQwiN3Dj/r1PODBZWa4PTJI9KXu5xQw34HVr/asKnPA70bnFr71v8fQWX3BAnpnMTYdw83+xD4mENOTN6cqthHEgxGFDnWSWs3tXRwXE37JyPCaYq52zrNFeVByEkkzIl2jItDyWVC0rTF4GqT581m92K8b4d6Ahz369iuWFJ/e8dW8xnpTqYTWK3BIrAbjfQdYn18ZqhiK36pzWmyPTmmrltGAbLHhcCMSTKMJDzSPKUBzXyeLvdTkDRmjyTL9dRz6UuKYQY0MxUL2pqXUtIti7+qjqBWEucppTYDiMiOqlAhqcGdpV4FjxAEx+oqpPpWEWMX2hUkDlYzwaiMJPpdfCjpj61BHAIs7m2Xqd2jBko9xNWt/liWp8a2I8IbkGltqMWfBXHfQtTGUfw9hmHcZKCxQG22AmnzjsRvP7eq/WnXnO5XGJqYJ44o5bgtnMVumTrM1vCcesrlMbkXvefrEe6P3DucXngctTeCaLO+8xidN12ONMc1t/qnPsznPMoWj0r5yC9mo+2OmWJWJxNCJGsoAT6xZLtMLDd7uj6noUnY0opIQg3nrvOsDO1o2tSHXR9R9d3k0RSB06VLqq7fpU0JiuFW6LW6zWlFHa7HTdv3GS3201r/qBz5rP6zPW4w217zmPuA4G05PD/z97bxFqyZfldv7V3RJxz783Ml5nvVb/66Gp3WzSSaUu2hGSYICwYABaSGVkw4VPqCZ4xAEYgecIABEhIlhrJGE8wjAAhBBKWjMWgZWOEsAUy3XZ10/XxPirz5ee955zYey8Ga60d+5zMrKouulJpFVGV7957TpyIOBF7r73Wf/3Xf1l9Ti2rOW0MKz6b/ioMq7oDreHdjYtBGs83hDmXpDe/yO61xAPrXJu+UJxzjlptZ+cbAee4zjEF3bkrLhE6TfPZsyiDQTEQddMDTjn3kC3Iaykl8mxkvXmezAPyeqKcs9ePmQdTaqB4bqZlIqfJkwJCCrGndt614GwsN5c78OfOTzCv/qC2D8NT0R9tTN5G4HmbxzEOineGPu8IgeI91XB5cYMRj3dIg+rG8AwgTtWyOahwPB7Qpuyvrmlq6vpNtYOtt7e3ACbelBPX19ccTkdOp5OR4Tz0mAa91loru92u0+yjOC1Co9BQWZaleyxG4T9shK48sU37TRRpNCAB3I7NyZStCC6AX3WQFaXXK8U9jPBEfYWMYssgammM/9FQsBmTjXRG92BGb6r56/F8t2OMoez5QnHJ2j3j5FyMmfFexPVEKCZihZbhNcU19daqw/eo1E3zxlwGEMxrJdjalmEa07+jUR89RitE9e6YtVq3HxFynsFD9L7IbG6KDfkzzs7PdvvgPJXYLo3Cu/f70Z/9ccc4I8fZK+8+QbiXvfBsO0Zzdu28LORp5ng6cVpX/3vydOdW6KWqHI9HDrevOZ2OLMsMNE6HA6rNKN66EaQOhwOHw8FWNfdiYANvTSJh7UZkt9uxv7oi58zd3Z0bptL5DG+l1Q/fNbCYYN9Oi4dd+ZxQ1TEPzo1zhDJjOCPiwGZkbkYm8zDZx/ArvMNxco0eVieFcf6cA1TfvCe5eP/tGZE4Z28/opaNCpwp+ldnpxK0aniI3SczEipm9Iw2MHWAu1P/PbOTRHpr2FZrF8OKaxgr1tuQtk4pMy8L884oCTmU+4bPpWQN12hYhfd7DH8+DE/l97Gdryxn77xzv/G1iDHHWPZy3w278dXYrImvjs1JZDY4YjJZerYiczJXFri9ewVJ2F3fszTvWjmsh847OR5X1mKG5TrdkFPitK7UsrLfX6F57kYlJsfz58+5ubnp1xqTf8y2HA4HWz0dSwGrWN7t9w4YtjORn3Eyj+zOCLNUleaDdcqZVU8+8IUpKmPd5cYTVGNLz379vaG6/Xeb2HZfJW+tRjYDFR7jZpzs5S3rZd6SLwmRrRk8lTDorW5U/EsDM26RaocxFBPW03oOaLuHtsyLg7WpA+1pYACfvJo9zpP9Xq6rg+S+eMDWSbLjOYPnBw2ZMjQ1SkKtVG3mP0/ZUsnC1qTe7qw1Nfy5C38utrdiIv2vmPLnsO3ZQHKrPnoqIZ2oIy/ljTP7iue/iz+kFJgZYByF6C9sl5B6qF9ZV3OFr2+uSU6Qs+pSkyG0lGcFKnnKLLt7rNVkEJLLT9bavF+yMWXHQb7f763aeFmMqFaty2D2jJRhB86fORrh7urqakhVL0PGZ5OBBGGahobfbWth0VfvZl5a1AKJOH4gwmguGL2AwDRa649MxPJEHYBkw2HAwVYdOUXSMZcezjaF3kPZ92XDTtBRU8TZwP2c25NvTjCL84eIdxipaTYMqJS1hxv9s60xzVMnwgWutIU7dN6KeFFpfL54Bs9OrEzz7B7NNt6Te4mxtkma0OpyGu4NhtdrIY/DfWFQJLhD+l5Dkg/CqPT4+x3vgY9NjQGR+jsByMFgRPz36FCr4z+3Pm5ausJbhJxdIYxwn0PvE8/ySN+/I+0CghmPRPICwImr63vs1FaV4/Hog7PSKI70G/9rmneQLCY3DMGK9aZsjyeA1Ah5Si3U4wbWZveifE2nVZhyYl7MA1jX1fEcmyDh3SSXLlTt2OkZngJberm2SpJMU1OMO52O4G1LJEq3fTDXMCSMxjiK9txzSdF+A0C6eFQQ/yTCUNd5JW/ZG1WFqEdi81JoiqggKr2ncnIA3YyalyP4ea0EQDvw2SLkacrkAko5mQxocs+vlG2sjT8tYS1Utt5AKolG7RkxsElfo8I9mVfB5IWV0Ds9intvgTdqbUitiDaS+m1REDdaTXK/ly18lHim0hgu4We+fRBG5XI7C3H6b1HteQ7Mvgs3sdSwIuEChyvjw7X7Kfp2gxYrWmum+xmriD03D4Owh5sdL5hyGBzl7u7A1dUVy7Lj6OSyUirzNLG6Wv/eEX9bYWv3DPB4PABYVeOnBA4TLTnCewkQN1blKOKTZg2oUMsGTfPSuyTaJC+WWXBVupTkLIwCulsvXkgZ5nnDK86L/ZLIJgD9FtzCqmw9DSxBUttS3N3d12ZV4HgTrX7zz8OV8fhjE7RYyWOcXIY9Z14sdi4RXJPGwrmcM+vJqojNY7yUZHToXoLPlPq9Dw9Nq7oz4l5SErS4PGSePEyyu1pLQWojq4Gy2pozcCuilvUJgPpyrJpDsz2b8EBt39S1Xt7H9kEalTPkm9GTeTMVGdtobMIbeWMbDEjfT7Y33+ktqVUqJ6CKMk8eBkjqnQvNPfXG4hhLtfp7ASguC1YoWKee6jWdlNaNhCRB9DymDmA2VvJ5njmuJ27216gaXnJ3d2dK7dmAwbMy/mzu+/Fg/ZuDFr4Z5uKDcFMOi89vLN4hbe+rOYQxlDNQETHmcIRYI2jawUo57/PcXxNP2avjLB5ijWNixGr6OBHpY6C/ls+/y+Uz7Z9r6hmmAbz2yVvbVlT5tvFhBsg8Wh0WH1RNc/hsDNWz+yoiEB0JmzFltTTaalmeWuqWFkaoWaEFOH9eTW/gcCKa2G+eoY3x/B4DoA/SqLzdU9m2S3LW5WfNqHh86f/CoBjF+Ud4OH313P4ObOXc2IlzDCYnIVkdxoQZnP31tRmO28J+f2XHSollmpncOKzrehaOzNOMNUrf5B7PCVzW40cVpnmhFvOWpjyTZntvPRWmae6D1jAXk1aorZmC3HqiNeM9qLvrtdqkGOULVNX3gaKV6BcscGZwIpsRz2aa7J6IZz3MC5neAJ3HTNTk/YpSTpS6bvdZBpzi4jlfeiwp0bNw8ewu65rGLQSWmjSXzNy8JRFhLac3vJtxwoZinIjXctW2qb8N47RjeLoRDkWiiNHHclPW44nT7QGqL1Rxuc1mgiSFHB70Ft6Bk/uS8YdKWf2cLjGZZ2Qo8fhZbx+kUYntEqB9m+u6GYZzA/H2fQJT2YBdMLfXvfHt53YgSw8anOIrgWE5tVZIiTnP1prBV5xaqvNFzFgcj0cLO1DXYzERahy/CHJc77mD6946r8WuUbqeB2KZiHmeyX4MqxmxARbaKkHhD8WxnDP7ZTEjsZp4tWEOGzgdRiEmRJ6MqJVqohJ0cyFP1tvozGtoI8ztcpTVCxZLQXSrJja5zYGb4d+r81FG2v6wUFxmlM48jtFmDAtAGOdxUdjwN93U7l1NLaWtL3UfM5xr30Z4HNeq0ePHw7k4XzcsgjWck3R2DaH6VtbC4e4OarXKZDCKvYJkC7HJINmzYAGYsxlnHbRr42fKCdJEyu9vqn8QRiU4DfA2AIyL17e/z7GVYb+3eCFnhiXOe7Z6jfvSw6IIveKlJELOycKMlGjVqO8hO0nO5CmzesOpaTL5gVorujZq9QExTey8mjhSivvdziqkPYuzrqHEbgZrmpJJDIoguw3HuLq6IvCW+D6Bt4SxPR6OA4chOXnOdVsRz7ZsGMRYB0RgBp4GC4xB8kafb62RB49iDHXimGOq1AzKm+Fs9wyTYDbbjPDoGWy4yyX5bXuu6JtKbvHeeRdGa9KWzl6vXAy/AUM6v96oKwI6X6a1RqnVvA2x6mdNVseFak9tq4dIx8MBVFl2C7K4txfn0y17JQnrF9Q2nMb0iGfSlHold6u1YziSTHPm//dU+PG4if39jt/Bc/P2sznPBILH8C6DNRqiyDVs5ia0L1pT1L2GeZltlXL+yuTaJP4lQLYWC601DseVtVZmN37Z61ju7u4opXBz7x5TEsh4SGKVqymljpvknHvPGMCV8kf1dB3afVg2Yd/2nE6nvqqZwJDp2YahiZTrqLMSQkQ2WDd3PraePnUuRIIuU2k6Kudq8BAGb2Oijvfewq9z7CUoApFWHkOQ2AI/uMRZOjdkwJAuP5cs3hw8HjNgZwvVcI0jCB2hsPvKZjTaJkAtItTVvZ64j8WMV4SjscD0sN1BVuuEuElSol5vNkXdV0O8y23D8B+txltJMhQUegOz97V9sEYF2OKR7if8+G1za92weDuEyAjoxX5vLEmXl4CFPjkLKW/1Ja1VmiRyMj4BLdGksex2HA7BYK2DV2D77fb7ji0EQCvQldsOhwPTPJ21LlXVngnqPYAuaPrhrvcMEhEyWXOzKEqMkvoNJLXjBWFvnEDdhR/qSsKrbK051hThybl3aSux8XlGj2Er599AxTgfgxEYvdCUsoefb46B0YON34MT8zbvd8TjOlYl4s/iLeNItwUu7sdmUHzf4RyShKQGeotsje7jnOWCMXvv3j1jKYdMJ1tZxhTC157uNilKq1y2LgNAwwlwW3YpQW8Fm3JCU/pxw/wPdPtwjMoIjhJ1NuF5XPoMb2ImIxgWe0b7xyA0Rdw53uB33WvVwFLEwo4pM+fgrWx9VFrTTpDaTaa/MS07al2RKfWMwjwtVAdCx0rg6itVgJ/aTFDpeDxxc3NvA1sHd39dV+rd3dagCjpdPnglW/jSOByN+r/f75imjEg0ZLfJ3aoNWpm2UKHH+xrEQRdvao1pmrvHF08msKJuhLzbYPBCtrCnYC09Nwo+fo6YaE2rdQEIbECki2uHoRjpBIFv2LNP3dDF5B9rdMbQKYxvLZXopaOevWqt+D3a8JsIzTYej481z86ktKVum3uuOWdkMiylrKs962Y4TvfSWqMlKFVJbegxBX0BUlUmydS4brUFDhEmsgfNEBkzM9jugUpc4/vZPhyj4tuA5w//Hf4+M7kxYM9XKjBSkKiBh95O1zAD8F7JMa0cXI9BGysOjYySRchJXATNwxtsdbMudJVWIU9WhRySkNb3uGC7CxVhXvZ9gFy64+EqR5uMsq7c3d6x2xlY2+LioQvwRKfDeZ6hNWSawI9xOp22HsNi1dN3t8czkSYR9ximRNOCNDM4MYlyzpzWQi2NjKHXQT+fZEKbq7VJeHG2agp4KGjM2V5J695UDg5Fax24FGBKJvVUS0VrRZYdpRkYnvPkQlfnYUh4pM15GklCuc6AYRkMCiJnBjOL1/hUIy1G+KF+UEV7+NoXLDFyYixO8U9kw2va4PWMPXjS5Glk16wNXZzI6ol4XRmjROh5wzEUyziyaSerbil+cHW4ITvXtJoC33vaPhijIsNvKsaMVEfSt+08tr2sSh7+cPIbZ/v3EdCPZmGVSICx/qDUMhdJOJsAjUaTQZ8knqqYClhZN1FkIaMpVkLthWh9xZNNHySMwG63M1zEy+rRyno6kueph13mt7muiA/iUorpm9bQuN28DG1G40/ZBlqSyTM4ing3gFaLDcJkJQjGv3P8wl3q0+GEau0TZFlmIxY2RbLzLRKbVqxfaym1c1HO0qnaEDJoo1Sju496MFa/4oGvSDc+DOFt8hzyGH6EtMOlfMM4inqKGSOcJT+RdkxmqFdi8/6E1O/3mK6OSb/dcx9LaehdNIDWdbUWsqO31TseOEM2eD7j+DXDFUZYGOcD7o2ompOYxZ55babj87bQ8We1fTBGJcyu+oqsHQc5j2/H32O7DIXe9m/b1x+FdWUihJDx+hFHxAJjtSsy+Xxj1w4aJ10/I2VEBdS6EoamRs6Txfac92KJVHGtFUmJeVk4Hg+c1pMBsZOtRMeTVSUv8ibeANX0Nabszc7rGcdit8yA6eCWslJKM9p/az2cXFthWcw1r1VR1s7dIE9MeWJOibW1rpVrE8nCmJjsOSmtFeY8e5tVAxjNNp6zWbfQwyjw1RXS5tlEnHMWanWDhhC5fPFn11rwZAYw1xeYaBBvwykM22YcFboOLuqEMw+/zj3gEYPZjEOlEupxYyYNNqGnccFQtgJI2DRmJBY9HwthkDqO4hiP1s2QyHZZnXE89gjHeTKGJVkj+O7BiPD+TMoHZFRGRsoQxFwYlMHrYAPb+ufeGBy88V7/34VQdnKOh919b9TuXQkFet/gGFAj5rDMC60Uy+S0up3HQy4ZBmAUB5rxyH2QT/NsqWV3hwNkLWW1hl6ehp7niZwt4xQdAuPfqpbeFg/N5mX2YkfzRu5OJ6Y8OaZhHIhyKlvmpwEuIlXKis47pmn21b9xOhWW3cyyX1hPp+71iaSz1bj/dIW7uF8j63bzvPAapjxwZI6bAW2KpopinkKAyaLSAVDVaLSVtsJRfwb5MnyxN20CwtYoTUay2obbGFs6OS42d5witq04cyPY2bO2+iPDyTbjst2nHjPZfQJm91Z6dXLeQlXzYjawObpNBp6ENpIopGy6taq9EZmNj59jRq0993TmNZwbF+37vav2x2Fb30+7Mvsm5bh9Rvxg4UKLexbxXkqJKb8pHhwDSFLo0hq3YpomqpqRUq+VySRKd883avrI8QjCWcTWZsgSybkstdYeh+dUSHI+uPt3coyjNFhPJ6KNrrh7fHAuy27ZWbg1zyxL5lRK17BtwRMpK9EuYpoTpRoukue53yMb3Oe6r/GdSi3WvoMNF+gTYQhhsvd6niMFW6PwT+lF/9rOiOb2fAbwNUBWRgW3zbvrfXACX/Hn0cH7blDo+4Nl7LpXItuCMmaPYEtZn/NnknkT1bIzrZlDnKdMkolSK6t7fNM0WQZKz43geI54vnF+w/Qiy+ZhU7ZaolKLdW9Mg2DWe9p+rFERkb8A/LPAF6r6R/21x8B/Cfwy8DvAn1HVr8Se4H8M/CngFviXVfV/+0kvpq8Surm6OgycNoBj42dG4xKvdQAP9ZUg/vfGWYFxIHpIJDYIojXnlF2XdlAcizRtORVQ45tknHWbk53PD9uxFtlabIq3vRj730aGRVQRB/yCAl9dEEibIt7cKupigpkZGJFlU5xbAkx5YpqSE9YSp9PBKo0xjGh/dcWy29l5ffauA3iYEJZp6veoZ2EG3k9MxFEJHpVOphtxMAN8kxtTCy83YSgPKZqxS6UppNZbfcQ9s+NI52BcThwZDFk34j5xVehZL8484E0y0wD3PIC1dLX7M56KbKS/sRQheD3d8KREXmZjXfv9GXGm5EQ54c02rPFd2+CZNlWv6UlIyNqmZH2kBG8O76US78xz/sFvP4lP9BeBf/ritX8L+Cuq+qvAX/G/Af4Z4Ff9368Df/4nuQglKkzDQdHN29DzwfI2z+TsWKNhufhcP1ccm9FjMZwibr5gMX3ySSpsqm1hBLqEoKPCrdWAfrv4cGvKOsgSntHF29YMfKTWTz4ZwtUfU5aq1i6jVrvWsq6U9dT1Sst6MkHmpqzHldcvb3n94jXPvnrGs6fPOR4K62rhR60Wit3e3fLsq6c8e/qU29evKEcPbcRwnePhjno6Ia35tQnROSPwHJM5XH3SV6K9ybh6j0xW4+8kosVE7qnn1ns0i3JmhGIsyPhcm55JC4yYisBZhiWx0f+71EHTMzwkrm9UbIvjmud6fp7YortBZ8oOxmWskzK7tWFyMZ7wMTmGKUFgHFPVce+AHo7H9xFGnHDAKBmpnD/77cd6Kqr610Tkly9e/tPAn/Tf/3PgrwL/pr/+l9SezG+KyEMR+Yaq/uAnOI//FiuAoRnWGW7DUeB8lbg0Iu/6N5yo/+1JZKyXr2BpRPrKlLLhGsnj1lhFR6D2tK5WlxEsyMgGINbtbp5odT1T8RoHWinFWJUpUUt1dTWvvUkesrXm1dAApu2itXIqxV3fIcuhyvHgBsaZuKVUDocj4MBogt1u4ep6x9XVnpxmDodbTocjx2Vhf3XFruzZX1tztNPhgDT1xlmJ/dXejY7dzzpkQ8C1VJsxVdNQaAhD7yKhZydCEDyMfNOGtEjRNkRq9xpiop49c3+usW3jo6HRphQ2g+LeTizeYyhj38OFwJtNyssqadUtmzKOxdhGADeOeXnd4+djLMV9UqzCunRMxwtMGegASc+MVzR7K/VIRcjLjinNjtls3/V9bD8tpvLpYCg+Az71378F/N6w33f9tTeMioj8OubN8NFH9+LVzVtR9YTM24HaS4MS22U49DaPRQmtUHcZ3bC0ULXxfXKKal86VyVc0nDx0YEY5VomNjG8CEyUKS/dIxk9FeAs9BnDA/G0IjXaYWhvgVlKQd37ufyuAQSbuz9R1iMvX95yPB559frA7d3RV91GnoSHjz7ik08e8/DBPXIyGYVSiokzp+Tykc1SymgP66JZ2Vq2SuuxEHGcQOEp9OZawz2L+zlPU69XMUPuq3He8BqxuM54GKq9ebs68Nlbosbrg5cbuEmEQlo3dmxc53l2LQDvTSBL3FsaQxLY/o4wb3zOo1EZxv4b2bz4TP+eYuNNXNQqzxNZpaeika1yXURIbWU9nVzSdNmM2njO97T9fwZqVVVF5PdtB1X1N4DfAPjWN39BR7cvvJIIgyxTcAFYKX0VdyQKb2rjB6GDveGGmwGA5ApyAdJ2z1BNdiBhjZuC+LabJ1S3lpTG2tzE+q3mAAAgAElEQVQwBBs4GVVTRrcJZw2dNFzSJEzTzmjxVV1m0iZ35zpMW3y9riu1aJ/YE+q1O61PvFq3WqSE0EpjPR1ZawGZKdp49uIFT598xfOvXlOakmZll2eWKdHKiadffsbzr57w+OFjHj96yP0H1zAVbtcj1JXrm/tMKXMSM2aZxOGusl8Wptl6FEnONIXStVNSZwmP3uUISk9TNuxCzICY4yBWUZtmgrBqq/SWEQt+yZsDTmLVsckmKSK4M28iKO/dEPv4iZStghP5gppvrWQd6QbOuymGlxHXNnYVvJzI3fhoNdDW2a8yvJ/6XbPvYO1SPaQsK6iD2GE8UYSGtkJSyGkm5R1ZZrS3bZW33K+f3fbTGpXPI6wRkW8AX/jr3wO+Pez3i/7aj93GCGWz7IF7vAnCnSmwj+6dDgfUy5/xO/RBiGujuAGZsjDnzJRgzqljB5IyRWsnnhnG4dXD2Sp+7dpTBwgJcWwHJJNMTLOlDdNpNe6HWEi1roVTWSEJ02y9f9bT2t1bcMC0NRLOutTVQoJWaKVh0rdm+MoKX714zQ++/JLb25dcX91nn5RpquznPXOa0HpirSulKU+fPefVq9c8fvSAxw/vsd8v3OpLhNRp+cE/SWL8PGHDe0ptfTDNczJx5lJJSa0/MRvpLHuxnno41VPDWLaEC3CxDoLVrRlge6kutxHUIiuXutqf1kbTapKV1aUa2QyKDt6BQk/Xglf5eiW3Bllm2MLzsiE3hNbdKG2eaU8nC+7VGlFRkF7X1Qk5as+71kJthdYKi7nM5m15KmkyYAv1aumcJ/K0gAyFmm9e9s90+2mNyn8L/EvAv+c//5vh9T8rIn8Z+EeA5z8JnhLbu/GRt2Mmmxt5Hl/bzzcrUv3g/ee4onR6fngznu/PnrZL7uLDNpDi7zOX2Jed5AzV1EvOxWuJEvggaq1Si4F707KwH0OYWsmzkedCMDmnzFHElMEUaNUMSmvUtUIxoe2TVl69uuWLL55RTsrDj+6z3wm7OZGZKMUG+KrWaL6lTF0b5XikfPmEw90djx59xIOPHpDyHVfXkGigFS0VTYkqlqWrqqRsRl5R5skKIWsrngGhU+UtBJq6c5hS7lKctXq1sESB5gZun4UbgyGBDau5DIfNm8D3KcbZcC+lf75tM63f9w62bxT7MYs0jrPI0ozXMxqQy9ciXDnLxIiYcFixBUvcYKqarIFWywROTtIrpbKWYiGotwPpTdxTNm83ZzSFQBbG6XmP20+SUv4vMFD2ExH5LvDvYMbkvxKRfw34XeDP+O7/PZZO/m0spfyv/H4v6E1rD531Ou6DDyi95Bhs4c8br9lBLaUYLot6licJSTJTHup93GD1Q6gxIOnnTt0VDUwlPhDM1GBtdk0M/xc1MPPk8gQeJtXWKK0hTh9v80wt1Zp+ixmyioOOmnxiKqdyMsNQ4a4oz168Zl1PPHxww/6qMC9HJhrUTPG6Fk2CaSyA5IRW5bgWnjx7xWlVqu+XsrDMGfGOAEwZ1dwrazUGr4BmqyMCSJrdeAxFeUQ6tLHkaXAs7fv3upkBjxizR+GJ9M+84+emJ7u1zRhDsHHMxLGrF0KO2IzqUM7q1lDEeTbtXMtlvNZLj+UMp5FBmFuSh4EeIkZYH61gRDDRqBOnYuFvaY3JQ+XoO9RUjf8yTWhyXZcYV6QPLvvzL7zjrX/yLfsq8K//NBeiFyHMJcD6lg90byN+9mO85XPdu2EbKDbEG5FnyimzTFYzES0vVIFQRlMLb6L2A4JL06yzoKZuODoPYdr4GSkZtdrGZO5M1XD/RcRab7SGOJU/OA1lXWnVgEBT2l9IeeJ0OFBOrz10aJwKvLy1SueHD2ce3MvsFpN2XE8VlcRMggqR8p5EWFEKRtdvVXjx6kBtTziVE4jy0b0bxyeMm1KrV/GqIJ6WbU6dL9Fwy8MQzVYfpA0DHHMm9bRn1HhFFbB5MJLE9VjAZASEnq8bjcegA2sh4Tng2traPYQYXNHSAzFiWnF9E/EsXgDKUWbR9LwHDzg/Jp17KCNIPRrEcfyFYdokKt6ecs8pe+bHtHZOh1s3HHPvVNlqASqSvHnYNHej0gDrDGGe8d8PmMof+Bbeg+hgYN4R9mwfcvs7rAabXTp3bWPrDx7jRiSEnJQ5J+bs/XM0hG6kA2Y4LhKZlbOCQrb4OWL/EFMy7HGy2Fk2KYC4wi6UDL3P8NQHnJDniZQaU557+vUoJ0Rad8vLqXCaKmVV1tK4u21Mk3L/QeNqf2RZBFiATKmGF1Ebk5qma0uW1Sh1NXJctoZVt4cD7cnKlISreSFPGyeiSiWlmQCuzVXPPftkYHXr6XArOIwJF9KYWI2Xnq/49nci8H+RzQDFyhOLQ5+Eg7LZGBpFb50stgC0VjqWFh0bDTcyNq+GJIMvKiLZhac8IdA2oxGh9ugF2Xte5tEvecMHEXrBYWut9woYAV4rlLQ5cHt3x+l0IAFX19fMy+I9l6vdkZTQ2ihNaCSyJCdsJuf5SCeNvq/tgzEqwFnIsg2f85j0LE6NOzWi7BoR8XnsPZ5EHGzMWdyY2M/Q74gjdNBfTVRZdeskF9eQkniL0u06owWpADnPzJNNvjPV+CRdiElEqHqeSQCMGamgyUK8VJNTxSeOhxMtW8fA07FwOimnIpzqSllXHtyfuL5auX+T2e32nFZBpZI00dZCOh2sw51kXh8ba2nsl4m12Apn4uDC3WHls89/yH7Z8fEnjxwTGJXTlJQ2LCiqsaO1iLXaUJ9PQcHf6O558hXZIJqz53rJBblM+cZ4CCMQ4XDqIU5FqxVJmg6JhSyBQdRSmFK2Wqp45pdcmDbKRdqWhkLIuL4RNO6hsHtFl4BtGBSgM56Bro0T5L9SCnnKPNg/sEJLB2i1mqHOyby/VhsNx6d8YVKG0P1sNv3stw/LqHDuVeBhylgkFj9jlZAYuPGR2O8sph1WOBFEGlkwD2WaWKbElOidBkWSN2Ly+g11ESEqklpfRZVq3Jam5HnX4+XOtG3KtDN5yZGjUWvx9PCJaTKPJvdJqC5T4L1w1bIotYOLjTxl5t1ETeJqcoWXr06wKK8PX5Dzifv3EveuEzc3O0QmKsKkMIugCVqCUxHuysSTl6853BZ+5VsfkXPhcCpE+hTJ3B0q3/vBF+yur+x4SXqqN6pljRHQrBp6mmwyVUXbjBUoFsNQhgmb00YVuARaN13aTeh6DC1iosY2Mmf78ZpR6k3qwej4zcmGdv7QG4nvaqppAa5vgK+eeS8RliWRt4Rb3hUR2bJaF99t5CL11LEbs7KuBPA/7xavvC7UujoOY9gfOSHaehJhmXfkPA/ZLwXdvG39/bM+furtgzEqlx5FfwhveX/DTzYc5W0YzPba+XuSAjRUy55UUFzTMwhO4nT+mOh9FboA8WKFYttHVVmmuTNAS6ne59ZWD2vGHl5K7qptsYKnlB38dbmA7DG7e1FaCstux8qJw3pERagivL47cTgduH/TuLmZ2O8WMpP1yJsgo8xqoZ7sZg5l4fY5PH1dmNPOUuh5olFhFVoRqmQkT7x4dcuXT75i3n2NKeMFHlvh3xlWQXZV+uz4x5ixi740FlKWcurNzCOkKU4qHMOBUa/knePBt6jiHUMK3EsKhTmBXiYRSv7BlRqf6+iN9PEzeMCXNU32/YwJLoO3PGIro6EM3ZqNSGmlC4G91RZFh1bJbixf6ZhiDgp/ygNh0EM5haqgqb5PR+UDMSr+hS8HjIGjA/jqP7vfET7zeMfUSXMXgy72ST5Ip2QdBUWt/qPREPU2niIkDEgNluQ8z9SyYvhJpDH9SsKIiGyyjq0aGJqEnBfnbEStSwzYoVhsYJo2bSayk6Kq12NnNRd5WRZTfJtmtCVu71ZaE+4OR2or3LsnXO1n5iVZKXyaSW1lBhYaV4DWRElwrMrz15VPH84s00RNFZ1mUqmsdzCdJoTEXTnw2Wdf8vjxI+7dLN29z2lCqUT1FF4oaL9aceS6rkzLrhvVniUZvMcwHLQ2GPBzDyX2PX+u9AxbeCeo4yc1CkAbtTKIMNlwSi4EFYtJaK5ItMjw66peI9T/51IF0hpTXgw384VFBgMa3kEYrf58h9S03SfpbOlQ2o8x0tgMF46NxPGaNiTNVjc1zaR56vozASLYrkGCez/bh2FUOF9tFKsuNoKSvsWwGLCbfJC08bN6ToYz19aOmPyBWJYmmdaGCHNygEstZo1BnV19KxE1GaFqRr8IUfN46ukE88wym3qbuZyNUq3517IslmuqCq4xQtpWSlNY27rLCZZenCTRvJ1qJDBUlSoVpHF1L3M4NSR/5b19lP0+sVtmpmXi1CpKRTB5R5WCFFvNmqw8fXWinRIP9spuScj1NXMS7g4nKK9IxfgpZZ64fV344ZevuL7+xO6nkwBLFZomhGyegD8jAyAVSY2kOng1DRUZwqfNQGQR6zU0YBAxEUeiGXi2J9qUtobWgoJncxI5LaiujrVYGNx0yPx5OFOV7hVmwYhkYsAnzbJ6TYsp+SVPo4v0dHDHcwid4W08pmQasVHiMXo/Z9IV4Op70sd6K3XDo1w2QUQQr0cLWQNSMn3huD9Ne/geH/451Ki98Couwphxe5vMwduOdgbSimBR7qY7kR2EFYL2bVBdz82oujhxtEOo3V3dQDwzHJImUKsQtlXSeASBoxiGop1MpeoeSHhVJivXFfLHdHWEDn31HwawAbvW/iOJuFGxOpqckq+aZgxDZc3aXVhMfndQnn51RBCubhp5d8s//k/8KfYPfon7D7/Gf/bn/yOO8gPqemQ6LeSsfPbZ53zyC/fIeWbKsvHHBg8ChtL9vBmIWD2jQK9ZbtXCt9aQnDew1sHrCA9U5Jy4xhambLiFqdz10KGG3ks1mQkduCViIKe2Blk5nQ7W/Fycg5QSycWwspiwtHUgNG/WgCkoWt64pjdEnAAt2lPJ8ezeBkSHUTJ27da9slK7R5Ozh8xebIhIz6KNnthbptd72T4Qo/IW49ANAu/ETN75WTbVLjuEPVCTJIAcrynWnAlXsRhcc8vCGNXe9EQ2gye+yooP5JZaL2OfvZS9NkVLZcqWLTDAsGBn9xUrJVuBVKmqTKqdXGeDTj10tlSohT+zZ1YMID0eTzZYc+qErzw03ZpSomK1TE3V+jRL4qCFl0fh9W1inhMP7jf215VlyQgzv/SL/wCffuMX+f7pcxqNVK0r4d3dgadPnnFz/TXHlzCDXRu1tt65UNVTnrIJHeEKailneo+emAxwbrhT8gbzdOpAhMOBYUSiOQoFA0cBk6Ycw17jFlmBYjBhS63UYp87laPxlJxHo8FSBaokpimDCnU1zk+kjRHOmnfFohcLCDigCz3rM+IwGwA8jFs3fDKlnsKeXJEfGYzQEOqcLbLSR30/77vmzs9i+2CMCmyGIwZZeG8/CTh3caQ3/mstN63aOAlb1kiVluhAa9JNnSxPqcfqQF8VQqEr4nADGI23kXqDbIxCrWrq6SFKDaQUA24D8cIbOp1OvT+P6iZNODbKCowlZ2s4Nc+ZeXE93AjxxIl9SaAa5jHlBNUFn0R5fYC7g/DgaubRgx27OfPXf/N/5q78Tf7W3/obvH71Q6Zph06NNQVpT3j67AWffv0x11cJkQzSaNSe+bIt8KnkIOhWDIjiaeQtrLmskRnBzABwx+cemTZFvS5pEz5qIenZyWjY9SVrR1JLsbRtDWFuq2mqbeUQ2R6X75z8WbRixY9NQYu1GYn0bQhnhwfUKQHR4yfKDy4wtPg+l1krG3tWF9Ro5hm1jdc0Fth2gxER/4X3HZmtn2s5SaBb3uhRexnuXJb7j6/b75vHF4bFAFr7XdXoyyn4EereuwSegYN7hTwYAKDjHQYIjkaueUMv8yiELUcdnJTwgkZ3N4R21H8fO/ZJMk0XVbEOA4Tjpl20Jzyk3bKwmydOKOupUEpmTtoHe3AbKJWShJaF27tGOZ54/OkN969nZjKH8oJaXvF3f+v3kJaZRbxy2DwiTRMvX91xOKzofWfK1s0IjG69qsle0VzGEe2Ac0757HNjpe8ZbnIRJnQQMngerQ19nZRSImRsplyHg+OSKMVeo8HxcOL29o7j8cjx7sDrV69ZTydW91Cu9ntubq65uXfD9c09dvs905y8vaigLdkikkxkOon1jUbEwWGIRSNCoJGgd8ZpkXNlfjCB7qDpW6ho9zDal2pVayLWF5rh3muHYoZNeF/bB2lUuovbsyznBuTN3bf3uy5JB2gDOxnNTHcKh1V0Myh9YA+EtUiD9ontAxxVj/W3KlRreG6CyVsotSl9dTEj1wYJScTzFhbavR3LRgGMxXVO5hI719V+x/XVFc+eVNaj0MoCs2NFaoQq1BjLyb2q9VjZz5WvfzKxiIG5S0owNdhVyrGSFitZuDvg98Z61NzeHS3cyWpApCQPDceUewhfz0zTjN9Sx03sRmqL8NB5Im7AzTNr232PDNgQEll9jLF5azMMJQxKLSvH44Hrq2tTwVtXDrdH7g53PH/2jGfPnnP7+o7Xr19zPBwQ9RKJaQKFZ0+fo63y0UcPePToMfc/esDVvT37/cK0eNfHSZlmpxMkq2iXnM7GCzGeZMvwRabrkjSXBi0WwIoD2e4Nvph0Tx7c6GyV3jYyxEXbN+N+WT/3s9w+GKPyRujT33i7d/K2f5cgbhwgevaIP3xrDJaYJiHnTfoxPteB0MG6i69AoQY2rjLEoJdNqczASO3u6jh4glU6TXkrXhtW4+C9RCFcGCbV7T0RYUpCLcIyL9y7d48HD+7zfZSyNpuskqhYEWApleyxdhI1YtqqPHqY+fSTiauda/mqIA2WaWKy/AcpWxfGoM23Bqe1OECIhUjULiEQPZzV08sdUwkeCLg0QQat3hzesylq4K6lZHHAtpkoNxs46zfL7oUb25YE0cTqtVn7/d7JZMLTJ1/x9MlTXrx8wYvnz1nXldZLLsQ80slEtVIWdtNMq4lXr19wd/uaT06f8Fgf09YdV/f25LTQpNJS9TqxsVI9QNPNE0tpA6jPCHoXYw7Y2muoetGn2rUN2GLn14Qninpbm20MbfPhp52VP932QRiVMxMQNzrixIsQ511/v+HFbJm57l1EV8E8+UM29NWV2r2quHs3m/cCBsZRt/fC20hi5CPJQsrzALpFnC/Mu93QI2hA/9km2Qi2xfumx3KeyequMobr5Cmz7GYe3L/Hx588Zpl3nI4HamlMeUetJ9dxdbdZKkJjSQs3ubD/euHjR4lJhJKF1IQ5QW5Km9RkK4+NAJdD5tPEp73dh2RUE8d6tLqTlE0rxkl7ZrBd/MirgFtz2UjvqZOjubz/t4eKkWKN+yPB5g1lfJNMEI3iS/Mc19PJ/l4Ln3/2JZ9//jkvnr+wIjwxrd18PZGTUfTTZDokpVlGJYswTQt6mri7PfL559+ltcLHn3xixrFl5n2hNSG3ZuA52UPWrR8UuNeSNxnRy2zRmPmRtLUZCcNgfYJCOX/D91TPw56gVii6iW7bgZBtffyZbx+EURnTuPbLgKtc7Ptu0Hb4yWiMNjQ8NGclOa08nsx4Erk4h2cZQhslMI1AxbqBES/iEumaq5KVyQ1NuPPaQwQjyEUb0pgo5pkYKSsli3/GcMKIWNh3IDleM9F2ysePP+bRo8e8evId1rV0TCpl039ttbjZVDLKklc++foNy1SQdm1+Saq0diTPmZyEUzMjYh5FojZQsZ8iVjh5rI0sE5MrtG0TSoi+ychQquDq+QE4ikQDcfXfzejXuq20NhEjlKVPlpQnosF6du+gOOBNVX7n7/0OX3z+Bet6YLefWXZXlqGjIUmt5Sv2nZREVfdWVE2w63rP1VXm+TPl2VdPmJYd07QnJ4VUkZzILgMRY0K9eNIucQNvVY0tHMDzGAZ14Da8ngFzQbdFM47TM0TicgkMHvuwGMdY/bnL/oyrdP/ulpIB3SQb3+WpbJvl7o3pKkhr3kLViGOCiydhavO4+5i6hXH3mvAk6A+kakOri1wbxG6ejZeZZ5UIXmkKqzZmJjQFuh+egjDNG1u2DSFDj60dhG1Ft0rpFOlEr0eqwhpEPWPJcf3RNY+/+Q2+ePY5rw8nvlaKZQ7SjGqFVu0+yETiyK9+u3J1raSWKLpiagZKpjGniVKhlWYQVG2oJI5NmaeFVrNzPrT3xJln+72RaBq6tbMV7KkZOMucVea8IFVY6+qGJDlpENT7Pp3X/UBKG2aCiNVH+Qhu6o9fG7VZm4rvf+d7/OA7nzPvj3z6tSvmfSLvhdwSOxLKLWWeefUayhGu7t9AVkq7Y06JmcWayF0J+0n54dNbnr98xs2DRyyloEWQNneRbiF5u9Hcx4e4FQwDEd5WkhEfOx//Z6JUKBoeq0pvgpby5llvPa10w6tiqa7vPwT6IIwKvGk0zjyPt4Q4bxqZcJzpqzEubZBc19NQ+Ro2YWjavoG4QfkWNSZtdtd1LDJTRxttFbHYtov8YDFxlslWWgopwTThRDi73pw3chxsJCnR82rl1pqxRXN2LopzYnKGbNddmzJNwtX1nl/85rf44ns/4MWL/4e7xzt29xZKPdFkpmKKaoUEx2J6tGRKowsy5ZSY8oI2oZ4qbVXKqaBMrFU5rdZdsbZKmkyL1ka9cXei02CplWW/38K9GOS+uiZJZvDFtFOsS2HqCm2pr+726bj39gy9aFMjRDBtYW2JWuD21ZHv/t7f47Pv/h4fP37Ao4/vcXN/R6HQ8sqkM4sqKjvumPn+d77icKj82sf32O8aU7oi5YW1Ktoqk8LNbse0XPHdL17w6vYlN/trmrqejgPQsUUoUmphkql7woGD9ADZMY+zLoRDSqEfj4QJXQ+Y2zj+w4D1NPWACw6h1vvaPjCjMr7iCukSzNI39x9/+qtmvb3lRhgOcV6KEYk2MC35JyLTYlnl1OP2AH+DxTVFm4jWPGqyY1rqecNKGD6rQHVBpJg4G2Bbz9KMqA5ci3aWNYpjikTtUWSLbIKexMDJX/jkE37p02/zW3/nezx/deAb9yeuJ+H53aHXiWhbrTgxLUDqotWKmsZtc6p9w7kcQtGJUzkZ4cwV9fNsRZOlWLV3LbV//60Vh3RPLbJnOQewnUip9XKIrn8Sz4HzSRH8n5S2gjsRoXV5A7h9feQH3/ucL37wGZ9+sudrj69Y9sq8mzhW0NxITcmtURHqSXj6UjmcGsfjKx7fW9hP1xQm0lIpVZFVkap89PCKQuLpy1sKlmoX90wChLXbNrBq0zmOotDD6Mvx3AH7WEwYQGlN1obDsboz/M1nC31ODGOXzRC9r+3DMCqDXTj3SN6eRr40KJvVtsnmTjJg/WUzJtJjRYTJ3OiO17hcoLul8VxNPKkX2zi9Xc4N3LBaNG+hkYLN2pSKZQZkwGkS8lbDEiDd5SrW08sORtqqVl0s2xqQpeTq9KpcX+/4Q3/o23z/i9/hh8+/z/3rOx59tOdmzrxaK2uJhmAzKpmmwqk2VJOHVkoSM861rJyOK7VlDqtyOFbyNHujwGY9jcQAb1XTRqENbSNEOkakGO5BE2BrxZlT7m78+OxH3kbPtCGeYsevkd7knqYcjkeePn3Ol599xSTw8KPMfr+iCZZlQqugU6UcVlI2L+n0Gl7dJW5Pwu0JSDPzvNh1ziaAJGkirYWkyuNH17y8e8WpWDiY8pYKNxwtQw4tYraUsG9bbRdn3xPCw/Ex6SHMGXfHP3eZPeoZJI3MUD4zJKOBeR/bh2FU2KzrT5Lh+VFZn01AyArhBHWVfDcmtC54HAArunlFYOC+wS2OzwyW/9L9jOuptdIk9DVMC8M6zE1Imrbsz5AFuhwcRqCq2wDtoNwmO5g8u3K2imFYxTQl5qvEx996xK8e/jh//X95wpdf3PFov+dqVkpWbg8rukivuVESeUqUqiSsKC/NifW0cjqtHI+Fu2Pi1Z1wXGF3tZCwMGRZdt403iQuTczb5CRzykbZz5NzVNR0VfNsxtmxg1It1R34Q2pG97cCQPNSxVui2MOhh680qwqe8sSpnnj+/AUvXr7ieDjy9Y9vmNPRvZg9a23kCapgbVjEsK/T6p5Y3ZH234Sra9graT2wT5ZZqlmZMswqTDXx6NE9Vhp5Wkxy0k1EnibjqYx4iWwZuzAMyqYRFJIH7pJvmMsFfT+MNF4LNHor4QF1b4dzD14ci3tf2wdhVC5XqXG7ZFmOk/DyBjpQjgEettpmsbYblpmMGHQLb+gNroEIhfBUJps72tPBF5dok75axiDSzMOA6PH+Wwxjb8cZq1eyxG+8JyK9LebG08AawqfWVyR1zEhVybOwezDzrV/5Zf7w53+Mz3/rb/Ls2ZF7HyV280SZE2hhnoxHo7WaCy8gzUoZjMUsrFU5FLg7wau7QsoTU8oIK/fv3fT2FYiw7Ha9U+EGTDtg7sZb3RooSl2N6h7Ad4Sfcc/NyKbeVrZ7d4GAyYiFwel44uWrO26PJ+u9NBvYO+UdmvYe5havUrYsYM7Zq44by3yPf+xP/nP8yq/c5/7Va/7q//hfkyoks29mCJplbu5d7XlxjDHhNV/z0o2HNVzzYTj26JFzHkn3cm1F3cIfNkM0zAR8ddvGUoxhkW0PtYXzMtz5+Qt/eBsYGynY8/dGN+7cY2n9J1gQlMSU8VO31HFs6cePTA84jV3ivSAYJdI8W0qZcw+lrzA+KYwxm3v3PtttwwnAa16anBmT+C5RqRwDp9bK8Xi0WqB5Nhq4X2+thVpbl6TsamRZ0Dlz/8HMH/3jfww5veKzH/5dPsmVBx9N3N/vjE/h4UMTw02kNsSzKrevD9wdK68PjUOdeHkoHE+NedmTBHZz5uPHD0lT9qZhFpZFA/ne2sJ7IEEAh1sKPkKf/voW+PXftmzYdq9UoGozg4NQVtOYPdwduTtWDmujygnEjeB8Rck7VKydiYowpRnB2nYoiSkrRRWkE0IAACAASURBVBuf/eBz7j0ofP2PfGxdEqsyictnSGbVQlZTDJS2UmtBdenGnZS6F/I2/sgZfjZ8x5G7NG6x6MT4imNXjeAePPbseErwgsb797Zj/yy3D8iowIahbBmg7f0flUp+E4sxxz7Kb1x2MhZP381i2zAQvqqk8Ey2wV9KQXM2+pi+eb55yqZTIuFJWPyds2m1jo22u4Shu7CbHIL2VKOFClvjbsumbAYFjDiWk8kdNJdoQJS6Vqq3G3v4cM+v/Yl/mP/z/2g8efIdmhy5ty/srmarHSknRGcygjZhkonT6cDhduX2CLfHiRd3jWcvC9NiBD7Rym6eudrvO4Y0Ob08Ut+hU5vzyOMxzyBIbohPmpw79dw8s8y6bhmweH0DPjExrf7k7KY/e/aMU4FDAZUCeSWl2WQpJiXN0Ioxbmu1zExNijKhCKfja/7Gb/4PPPvyIZ/91j10XSFnpFSygjrfSEolnSptNY1f/Donf8aSk/XxUkC92ppzz3okt42vx9dJsoXGtt/W+jWMk6phhQFun/cZivXuzVD9fWwfjFHpwfJZKLSJFY+FhaNxOa/edRxcPB3skzSBpYEIm6AdJ+mALkrwCVCl+bEkaK3NwyZ3V/sq5IVsjUatK9MsvUFW171g80w26cLNQ+n/WkNXz2QMItnhFiteLp+N4p88bR1pVaSSZEJrZuZIWRoPv/6QX7v6R/n+bz/g2e/+X9w+fwHtimnJNPd4VKGsxodoBU7HSlkzr25Xnj4vaNoxLzvTFkmJB/duuNrtztKYgTe1barTHLRFjX1rk8lAzEI9m0wWdtI7FoxeSrzf7x02aSJTth5PHI9HGpnGBKlR24mm2ZjOi4Lla6AJQkbJTl4Tl2I8cbPcciPC6dlr0rzj2CqiFWmZ5uD2NAvlKObVBV9mKMOIGT2+FhIN56P9Ld5EiFhJNEI7/1RXgOtjv71hLDY6wmaIbB79yMn3B7p9QEblkqPikoCotax07MMXgG50vLcH3QXxQRcJY0UtxRztHhQbEH7ObmZikttefh3VsZmY/NnEe8SNUfOQQaOwS5BqbUob3hRMTXdNqKjOhHZpSkKaZnL0/JVNF6PUYqsVieZkN+M3WBMqVSssyzNMU3LmqSmoVQrznNEmoOaxPHr4kEd//E/w8lu/xO/+33+bJ0++ZC6Y+NIklHJAJHEsK2sRXt7NvLyFHz5bqbrnar+D8hpthXtXOx49vs+yXywToWoyApHO91Q44tIRYizSrNq5P60WM/rz1vw8pUSLCZqzCzTheAyQxGqJSFZdTii2Ca9u76hFqOvqbSl25KzkpCSUsgpKIVUBJlY9YR1HM1dZ2M3K6dS4uUqkdPKuq5nontImRWUlocwVjkDBMn45i7GPF5NybK4jkyTGpC+UwW1g8zZjj24lcUMcpR/D/oGdvYt3sskhGI8H9z7N9unPH1ALdCzkbOWOvwnzMIYd2oFXCItPl3eMx6YeKiRlw0+EwVCcl+0H4Jm6cTLzJFEcF2BwgGsQ+Bkqja7VijeEqhWVEyIe5lhJL7UKSkXFS/gJqYPMtMzklEnNVP1rq9a6chioQbiL795XtVaBFdJEVmhrgVbJ88yDT77OH3nwEc+e/JDv/+7v8L3f+w4cCut6QlFKU9YiPH+58vrQmJcbdtM1SSvLsmO/u+H6eube/WuW/UJOqTM8tSk1VOo9bOsNscS0QeKaW2vM82Q6vn6zQpDaaoqy9+OxcoXOHfIukuJYle2/8uLFK06nCs16C2ubaC1T64l6OpF2O8gzKSlrxTwBUSCzm4T71xOtrlxf7Zgnqxhv2pgw76p1akGlrkqrphWz281McyLP2fgo2bSN+6QXkBTqdUKNMMjfjJA3xmpK4pmuc6xNe3ZyyIKxgdfmsdXuncTfjmZ3D/F9bR+GUXkLTmHhwIBw86bBsZm8vd/BXa9IDuNxiVFtj3F4LXZSushxUKE7jKhtqKswgSfpbqjJ/akICUXVBZNxtbkAMn3y5OwiPP59ouCsNev325xPMmXvOKcuDMVmQEst3TMI3KbUYn2VaUzTzJQnjuWInryAL898/Atf5/6Dj/jmt3+Jp0+e8PTpEz77wQ+4u7s1QHZ3xbd/4TGPPv4a07LndDhSb+8o9YBQHYzOpByVsBtGdLm6BlaVkglKFe8omLOQ2+zh01bjEtKPY9r9jcEiXl+jJndwd3vkdFoJbZPSKqdT9ZW6sKQ9lUFOQRLURp4Suzlz78oM0dUyMSU3hkwWEjaBZB0GTkXRCq9fH1mWh9Zm9IwZrcP3j38DjiLbs4Mho9XHN8QqdRnq92TCMFfOx68lI8bK5FhsAmN5X9uHYVR4OxDbWz64+LW4RyAO6tr/t7ulLTRU7F9MwJSkl6eLbJ+F4aEPZLO4hnGS2NjeBrgIXnMR3eisUDE8I7RQaiNNhZld93RyH0TnmEGkj3GcKAnmQWg1LoSaJ7NMk+0nlkYtapW43fAJzPNi4LJGOUCilOrq9orkif3NR8zLDQ8ef4Nvt8YfLYVyXK0nc2vMy4KKclpXbl/e8uKHL3j16hnrektyyQhbEe27x/nG7xPEt5T8/OL304W8ay3k8Aaat7SYJhKbBs1l9kJRF5F2+cbaKGtFmJw7M5GnibW4FnApTLXQxGqYEu5J1QS1MufMzc5qna6XRPNaJKPgGyhcTsXCUDVD8/zlSx5/614HnsUXCWXT3hlD63gtJEvf5TVEe5hLAzAS2cafMV7VxyFtG/vbeA7D9dZT/ky2D8aojNub6WU2IxC/9rcCuNtwEvEkzjxZ8/LMVvU5fvhdufxtBTn3crpB8vPaQMFp+83Zl9KxkDyZp5FS7ryIeZoc3NvATRGrNFVnh0bolV3LNaVh8LTG8Xjsq/hut+uvRwamBmjqP5dlQWQldHu1CZoTkidympAG++sJWuH27g5oSDI+jGrlwYMbZklcXU88++pL9vudHbtFhXHgAef8iE7gi3IFH+jzPLGmzHQ9eYc+E1HKOTO1TMMBVH+uPWXrz1eJmhulFsec0kRSZVlm5t2eykqpjVYLbT1YODjNhJat+HMTWfn4o0xtiXmy52ipZmdnazFsh0Rm4qsXL5l397i+vs+y2/XiQb3ghsQ43TJXLjEanum475m38qYXHfvHsc44Ls46TgqaI/zxwF/De3x/UpIwLr3v2ETkL4jIFyLyt4fX/l0R+Z6I/O/+708N7/3bIvLbIvJ3ROSf+kkuYiS//ahU8bveOzMWqqDRgTAeAo6TeMZYMYxF7QaIhzFZttTlmNqOv1srffIGNhATWWFzhXNmXhaWZU+eZpbZWp92wzCKKrfqnJPiq4uFP9vfPnDY0oewhRsjxXtk6do+tcfjyzKbqDONpJVWV6AZMRBlPR046QmZxSinkpimmf3uit2yQ5KxaSdnCZdSQOlq73GtQTvvz8XnSLxWSuF0WmkuURBh7jxNZ/IQ2lfsqCCPLIvVHTX3FGur3m4lMYkJVy+7HXdHOK7GIViPR5JCdsDbblkmiaXCP/3aFd/+5jVJCqqN43qiaKXUYt4VK9pOvL6948WrE48++QbL7pr9bk+eJ0+Jt0D9DNO7CH/GhezSoEQ4LLLpnowe2mhERmNy5kk73hLyHhu3hz4G3tf2k3gqfxH4T4C/dPH6f6iq//74goj8Q8A/D/wa8E3gfxKRf1BVKz9me1v4c7mdI9+jF7M9TLAan+zpOYmB3RN7ofQWHx2qOf3cAt1jGNPMHUHfxFgGAltwK1IXKjJspqG5bb1kZNAjFRN36gPKr9cATdNelWQSB1FgCJyR6bJzRDp3IWemaQYxCclWq622eUK0sd9NnE5G3JJkOivLPHFYT5Cte2FZK3WtlFNFq7KuBdKmSP/61Wuu9lcsi1/ncH1h+ESkh0TTZJmInBLTfk8pKzQ15TWaN2qr/t2acXl0e0bmrQEqpoDm9HxbiauFt0mQBnPK7HZXfPkEXt01bnYr19fWc0m0mVCUZgeYrfuj1kLSCs20Uk5roZRGbmLFkyKUtfHkyWt2N19jf/8h1/duyMvktT9GQRANSNfwj4Z6qG4WxpjcyevKtjE9ps+bh+2hoh/zIfaN7dJjITxewqOJz7//KuUf66mo6l8Dnv6Ex/vTwF9W1aOqfgf4beBP/KQX8y5j8uaO0FevHs7EjTcXxBijCg7a9sehA/h7ea7wVkS29VEVabo1KFNFvZeMEgxH56o0YzviK6yFPVvadASkW62UUijr2pu+jyQv+2nuuwFwofreXP5wW91SSqYQ743htRkrdcq5w8zltNq/UklTdqX/SnK3bbdf2O/2hIhS9rYU0b40SIGffv3rPP74Y8NK8kQpmxEZJ0evtq5esp8ChLbV1a5zYw6v69qLK20F3iZTa9bl0DzaATUbSixylu6hzsuePN/w+k45lkqJIlC1+2dU+gVlpmHyovZdE8cTSNrTPB27HoVWF16/LDRdePj4U3ZX91iudiy76UwSNLJUNiKDURte7vavlEEmY8g8boPbuFZOqLBQTLbatMv50o0JEWaFhz56LH9/ZH/+rIj8i8D/CvwbqvoV8C3gN4d9vuuvvbGJyK8Dvw5w//71mcvcDYuPns6w93/dcyDsi8HbCc80SKwAccONQxGf7SD7di2X1zaEEgGBKshAY/JrUAd9LYhwQ5dMYlGSsUXj6K0W6xPs2SDhvDHY2KoBOCPMRce5WoqDn2ZQ5jBamAdT3LhcXV11tq4R5irrqQGJda3umyeqVpZpRkmUBnNeWO7tONzdobqy3y8cjkdAmGYTr753c9/umUoHTMMYxBZ/z17iMM+7bjxr21z4ac6m0gZMKVPXwrTMZ+zRUfohuBpW1W2GRLVyc3PN63bHelpZloWr6wfcvXrBsTSmUyGnIzvJaMook7FtZcIyzAWRCZEdpSSOJzOSc5pZT4nb2xPPvzrx6bf+MPc/+hr3Hz5k3pvDKsk902yLAA0XAHOD2L2M1EPBrTD0nJ4/LioxBsL4RCg5AtgR/pghzz1ktPdAZKvy/vuhSvnPA38Om9N/DvgPgH/193MAVf0N4DcAvv7pxzq8/qbH4uFFf8kfjhKhzRb+5GSC0DmasA82IGLd7UCjh5M2ozP8swZO2WUowfrZOItU6V3vcA3YyVXQNiwkEUwUREyJLb5Gckr2ENbFw2/D9xevQbKVrhm4KhtXRaGzd6d57mnqrio3JajKst9Ri+MzusXjkSWwtHfrE7dpo7SC0phzJi8LUzJx6v2y43g8+qTetFBigC9B68euG2B/dUWt1VrTCx1wzS5AFROqFDMscW3V64laa0jF0+9RWyXslonT4YhQSNqYRbm5uebJq4Xnr1f2uwltyulwIunMvAv+CYgsgOmmqPfVzgqJRDmsvHzd0HTFp9/8FR4+/gUefPSA3W4m5WLfwWuXcF0bEw7GPA0naxo36k2eCZwblC2MjkHOYFRjcTvHY8bFOIz7NpZMOCuEwd7X9lMZFVX9PH4Xkf8U+O/8z+8B3x52/UV/7ccc8O2hz2Y0wOJS53XARgcXBYLsA0mF7H5D6j7GiI1E4GSeR09UqmLqF9s1qEbjdueLNjvXOXBm1ybJegBrg9Kq4Qi1kNPci/6s2tWMUEyUZVm25uIalaqAG4xSCrK6d5KT0d2rNb6KmpN5ninFCuRg8HBS8nStgiTmnJmzGb7ixw4C27zMRN+jsg4SD3Yj2C8z62r3ZMoTktT6C7OBhlvVrhXZ5WlBHcwtrUJNndgW4RPivY+m8HSUJJlyWn2CREdAjaFiPBJR1D2lZZ7QWpnv7cmTsL56zrIou5tHPH1RuJqU3I7Ms3I9Z3JbkVTIeeZwKI6hCaUcaUWZpyu0CS9e/b/kvT2sbdmWHvSN+bPW2nufc/+qu+pVv27TuN3OQBARkLREBIlFYokAYYRkAhMgOcAiIbLkyJIjpJYcYIk/SyDhgAQhERAAEhYSAgegpvu91++nXlXdW/ecs/daa/4MgjHGnHPve+u5nqW6utJb0tE5Z/+svfZcc445xje+8Y0L1hJx9/yEw/NXuHvxAvPiQZyFnOg7Bd/pONlUbY+yinwxg3R7YRC4XnskIx/nHZIb9Xa4t5ve+4Bf20yc07T4FWD8/R//TEaFiD5n5p/pv/8mAMsM/SMA/wUR/V0IUPuHAP6373LO0aDceitXKdOrxxXMNJkDpwLVDbSyQMeYjO9a6w6CXYdEzVNRmjg3pbFec9LSwY7AWvvjNOTJeUcIsWUG7HEGMIro5Jzhar1SOgNkNzaBnxYXg1BzRqWiwtGEnFLbDS3kGN1dwx6MC+JALdXbGlVpBkvGTI2O/l2z4BxkXo9+xr7vgyFxV+PR+xdBjYddl4hBgZSTY3iQqulbCFgry9iRjVc/PwDU0jNwwTtMWocEZkxHj3BghG9WcCX8/O3X+PL1E5wjHI4M2jPYiT4K0Y6Td6gFKAnw1ePxvOGb9IiUCUwRn37+Qzx78Qnu7u4xH2a46BGmoJIGmpsavI3bjXFczP25XscF9JoeMQajO66eWr1RCASuwqTR43zv5sx457Hv8/inGhUi+i8B/BGA3yKinwD4TwD8ERH9S5A5+6cA/n0AYOb/i4j+IYD/G0AG8De+S+bHjvcPyPVj/f+GeuqewM1BHLM+zWj8ipRat/adIzDyAoargWmSCqqO9j5JC6pLzqySiwVcC8grRV8XGdDDHPvbOhXadyT0z/beyt+pWT8GWuFdHhalZbZMDKrW2ghazIyqIKEZTNIFXiHGRzRyLR2twkpEjTkMoLUTvSXuiWC31KCI1XNg1+UKQL5prqScW99o730Lx2Rh9nExY9vGRos322J1QIyiy1K4IM6E0/2M033G4XBGXs/46uc/ws9fX/CiTsjMuCuMyTOiAyYPZd56lOrx+LDh7cp4/uozfPLp53j58iVO93dYDkeEOInwVPTDHLze/H6VR9BA9QGwv/UyFB68oQpQG5vbjNCt12JjNWaMPjpPhZn/rfc8/Pd/xev/NoC//etcxMhTGc7zrT/6JgA9Q2MclJ5K62d/zwdeuyUD+Go79i3GYdckN6kvfmO/cs0oBUpyC7DirqaMX6uEP4YhuI7S2y4li9jIcWI5RFkt4qrqlMxF7jKMed+lRMCwFs0AWU9gQIzBaLjMTQ4hIHiPlAuEiLZj21akfRcMRJuDN45OFd0UP6RGBVBUDwTmARWE4DTt61BqGciAXdaAmcG1Ypqm5rmQbgglJzBx864qk5AENZT1PoAmQmBGQYALjIkijgePu/uXuLu7w88/fYEf/cn/g5998QaPR4dPXkRMgYG8Y/IVe054fLqAwoz59Ay//8Mf4tnLT3G8e4Z5mbTdqdfe2op3qag6RkrCzfwdH78yPJpifl/oIr+v32fZxdvWsON5x88fDUyfrx8/UPu9HW03vDEkt/J5cpiXYuCsxJG3Nvkq0/OtIRDDuBFmWNp7hptvN2+8iQQhsYn3IouJvOALlSuCThKnqvzehysvyGqC2k5DPSUIUGPV9ljZMJgOBFplcHuNjmUFkKzpu3b+uwZoqYVW4K44d3e6w4XO2Pcd8zRLBbC+dt8kNVtKFx6yayqlYJrEM0slA9nGTZP0sgu0hZOzyFBa3+muLF97yFczclbDAuGplCrs5RADqhO6vnfSjjSwdEtcyOGz3/kErz69x1/6g7+IX/7sC3z99S9xfnqDtVZ4D/hpwunVgk/vT7h//hzLckKIU2PghuDhgo6R6tFWsPJI+twajbTNk3H+2GtKKdoPuT9/TVrsLFmdjGDuG9NtOPU++YPbeescXW0A3/fx0RkV4Ntjw/F5AN07geIn2j7TsnfuxmGhlqMmTQVf7yxDM7yr8Od2d3gHsScROBrxBDM3tVqbUw0vqrhJVmDYzoNhYirCRoTWwU9eK+GITCT9coTePqRWKQMAMM8zAKtP6sZi/F7GD6m1Yl4WGZ1ahc+in0kQ3Mbev+87cimI4drTWtdVQ0DJ0ggY6zRskradKSUxbKUCHm3x2BgyC13fsBVTPRs3FgHK3YA1aPjpvarbB6AkEBW4AHiSUHQOB9ydfhc/TJ9iSyssZI0+gL0Q04IuvOA63uSDbARMrH2rNcymbgS+7Rjnjc2xHt51D2Oc0wCuvrd4rH3zufWGbo3Xlfc3bHoj4fP7Pj4qo/KrjMi7hqYPmBHdNBgQ11JgRn2lKr8NOIz9LXwlIQ1Z24eRMHTlkdy4q9ArEHfWXam1WWN1KBclNFHkXo1s301qcjrGQbUCIcA7P3BXqhLgxNOR1qJZMYeCEEO7Nu+F+2Hnq5oCsAVr1zhNIs59uVyEK0KdYl9VkLlqynrf977o7XvfeDz2uxkwQOqhAKzns7RHzUWo7UDrGTyeA0BTjss5S9gx8FTEQHd1vKJdF1s4wYAgbFWyeyQsYVBGoYToGX6Z1YiJryvVzUHpABpOQ3sghYgKVglLNNEtBexgqm52jPVct3PXnmeCeCvMV/iWzesx+0Oghodd9we6Xhu3n2vP9Vn64Y6Pxqj8qlDn3bAHaKAqacMwIjgShqjtBIY3iBdgLE00TKXfeO2jo4QqN7g44+IXn0iNiwGIGrNWEHIWoSTjFlg8O+IGzndNkStPCCPg6TTrYS0bjHk5fHvqnIRSsshLeq+6JkUbw0uvoaAYi0lPMnPDWYxen1MCSJi55qZzFZkDAOBN6na890jqNXkf2nfpNUkFYqDlvFKUyLhcLljmBftG8O6IzAlhdlrsSK1tpy2MGGNTmr8CHqFaN16rkCtpzx1h1ToQKjuk4rXJVhXxaojIE4h7jyZot0g10uAu5UhNdJtAqEJT0ASMyS4QvetNjwbwfY9JSKrzAdfYSQ9r+zgIBeGmz89gPGx+Gih/i+3IOT6y7M+HOt5nOG4NjAB1Q9xqIYyyox05EHSnhTFutf7iJrQR022ToyPkbYe4DXEAEPpOwVVZjIDUrXivbUB6P2WSmKezIgmAFuKRcy37AQx9hlsI5BtpbE87mKt0JSS0kMoO2+VSSoghaHFexV4KaqmNSGYLFNAMDnVBKpHPlNYbKe2oNQOQQkNYDxsl4UlR4H61G5taGUBtYYIZ+7qKQUsJHCeVKkigEDAZnR9oSmfGvembCSmfxvf7IX+0ewZ9v/MASoELAdF7pLprj2MRuCY3IxBL/Y9MCQTvVEtXNg2A4EMEMyFzVUOiIYtSFypdL1o7xqLK27k9zmGTxZTHMcxx89ZuMqDt/zaD+3mH1PStMWmgNz7s8XEYlSHFNj7Y1OCgA2vOiRoMURrV5leOWpjjmmsr76kEmLykqepbbHuFnQAav8qU46ajKotNjJMJNUtXPzCrhACLjCMggJ4X5q2FAmwpQg2tgvcIziHrhCL1ZJqmShYmKxHBkyiN2YSVZvE07E5SAhC9B2pBSuJ5VaH8opaCGD3macK275IWdw5shYlEmv3JEm4RgUvVJu8QjRHnlRrIIJYwoZSKkjOCC6i7nMtXj7BEIbDljG3bm9cihD9hrkYX5XsCSCVjPizaWExGvgJwUVm0svJaGGhsVQlVtVQAQC1yXyoSnAOiI6nh8SQkV8WzGk1eK55ZSypkPnQ1/BD6JgOE7unaxlW7V3GLb1itVJ/ePZQ2/KzPR2VoD+G3rQeZn9w86/4+yNyrLDo+rL+r1QkNWI6c9Ndelv+sx8dhVHDtqbwPOxlfAzRbDWOoVtINldGMEUajT/1ctzvJWHBlVZ6kVbH2uaWowfIepQgBC43QVlFyQalV+CFVFOJCjJrJeRebsSK5oo+NeEz7vmr8nDQOuq6cRk8XG2ZiJDMjkBmtP4QgCzlOOByOuFwuLZa3ehwfAtZ1FWylYSr9DhDEoEnlsSzooOHWnhKKhka1VqSc4TS8OhwOzS0/HA4NGwCUC0NiJMEs4tWl4HA4wHvBuJxiF/JLVn2ptV3VOGZj9sw2AuntTDrG11m8jm3p+4b7VGt5J2Ni3uGIXY3z9SrzQ+PMvfZ8rz2KaxGl8Xzv83hGsP22nmfMIH1buPQhjo/CqFjMdxv3mXtok0ofbCLVjSNgKuTMaFKudlhYRObwqjM5LPA2KSDGxMIb5usUXuWKWnS30zjemnzBYn1H8Cq3KBmKquBiZ4Xa4nDet8UH9OIxMusIoVnnjIYzMEmLCRTXmKci1CSLI8ZOoLLQQRYy4en8JKr43MWmbVGacclZJCrB0mqklNw8Oecd6l51wemH6OQv1BemLYYYJ2mspeMJRw14NKlO+U5yjwrnK0woeA8magbwGgNDu39h6MXc7o0WGpo+i73WfncQuy9UKdTsi9UA4StjcTM/b9O8NufG19jv20xQn4bc3jQmB24zQ/80aKDjTsOlvM+L+p6Pj8KoALceypjtaS/ohqDdCDQgXnaHwShZzN3+bStNjM/NTSM1CO0DbgwchvPIoumxLJk3ApEMQOVWaGaLyDwIkR206yXBALQeBjCQ993J55ymZEPQdhdQLZP96rU5J+UlyOuuvh/1PkHOOSkI1CMX8bSqkubmaULwApZ2/MRKIgDjLpeSpbbHUTOyUXf8CiD4Gfu+YZrmFgYReaRUpJ8xxKMi7xFDRM2S5p7cdCPsbTdAFOT8kJ0jsqyLqPfXqjKkQ4bqdte3cXFuDHGuW77UatnBd1O145ja+cfrLPxuI/Xbz+4Ynm1414bvSoTJ3ein4F0PpM2Dm8+69Y6+7+OjMSpjrCL2o1vdqoamHxbe6KSAgraKtdiCHY0KV+vAB93JR8/IWIcs0fzNTbPXjWXnV0aQSEroSUWDdDE25miL4zvgOVYSm5Gx8498hNH42SQlcsojuRZnIiIFfwXILWVvnzHPUQoededO2l/IAOKUEkpKTcvXvEfDhJJUE2qLkIp926H94QFmpLSBwoTAUEZvRvRyLfM8t7E0ZiwxmleSNIOU9gTUCq+iTc2jcEZvVx4SrmnozN2wG2nO6Xyy3kNXm4p+Yw1wowAAIABJREFUQRvXkgsYGZbKvf7dw6tv81rGcKT9cGdnf9uiFuMuKe3RAL6z2X3LOUaj8j6P6Vd99vd5fDxGZdyZB4/iChjhwXCg99qR+NXCn47sw24s9BRNQ4dbqw3hmIhLLTvT9WTpe7LG2qPHNIZGQ3EYqqjo1yJEMh/jlUExQ+TJCQ4DAOSQ1f13TslcjK7vqpyUWgrO5yc4HxCj/HQtWxkrM2ghRPVGGOfzGT7GXqinhmVMbdv1iWfiAe/ARRYxinRJJDCm6JEVrOTKeHp8FF2XWrSyOon3hR0hTJrq7rq5DWDW7yXV3QqOxggGkFNq1d02L6qCkjKG1+GJV7Yws4DiAiTngZnKIFyn5Xv4Rk2S4hZzMaNqj42b0TtLdTAAhuu8+5JrI2HeuG109vi4CbV1Mfx+33lHz661mvmAAK0dH4dRsYXKrJEHD0amvvtadRnNEIAEV/FeZCTdzc2/st+G0WAwPEYGIWlQdevKmjWqlsJmwLwl76Q8X4yZ7lisnQaJEAdhaqAXyY1xvOzaXYynFIAVh0Hj0JAsPMeqyF+Qk1xPJsKeEuZpwjRNWJZFma2sdUNSk8To3o5NOAuHgN5GxHuvPXnk/TlJVmjfVjFytUiIp6LVtVSABCOZgkf0AYUtMxSR9yTKb6q+fzwdQc4hbTvIOwSSfsNTCFpwKIzinBIC0LRkWcFhIqdgNNCEuNruLrenqrGQsgnrmX29ozfC2GCcrkKYAcsZOSR2VDRr0FTsZIq9P0T5NnzFwsKRVmCffct7ub2G9xkZsi9rs3fg+XyI4+MwKhidkwFHGZ4UW6D9fAyN1cHzHhpji2dza1BGb4TBEiwD79xsef3ohei7ddJz81kAsKQ3C0txHcDqXWio5JQUVwoy94ljqWvbrUe3dwx/TFvEvkvVpu5JtU5sVxQlOGOTSgHe42PGPM8QYm6AKcU557GXgm1drwxJWzTmLYSAOE2oVgpQ+xiVmlFTEhGjlBG8cEJMuyarJEIqFcfjSTsGVtSsjFAwqKL1RIrBI+07fAgoWXg1lQt8GLI6AOBcEyYXISkL+UQKsuMTJm5dtCsptdBnbKc67uLM16JVdrw3+wL0cFVJiu8LVW/n1uihjOfn5o33w+5JD3ffNUq3570Fo+0abx//EMdHY1QAZQ7W2p3GG+NCLbQhrUi2IkJza/UcZnNsFxl2kDH00aG/mgjO2Y/yVVgXMCRcaWQ6VDEs2vyKoDsf9x0fxPAURe09SqMrIYndyBwQNWap9K6Z2nO1ClBqC6KNA7FmmaTNhVDWE0olTMsRznnEGDDPM5wj7PuGdd2EeEbGGpaFmLMYBxshaKhh6WrnHWKIyHkHZYb32gunFGx7BlAwxwUhBKRtB/aEw90duFZcLpf2XXMWUFdo+GKkLa3ebofGAlSoGWlm9bRsoeFa27WB2c04arYNwomRnjgyh4yHA8NASpFbrPfEjH8bf7Y5073XpkMjN73dEyl4RNOuGT0Ru2bbdK4NVp8Lo8EYQ7fReH3bYa83ofBvw4C+7+MjMSqsbn73LvgGmzUfwZHUZUghIXq4BHN7SZutC6fWDIBZ7gGGRH+XhE8iXOylLoM8RBsEAHXD4myiMIQ8RmJ4CgZMo+YWtrla4cOkmRjR0eUiHop1H3StaDC3WNq+U4ySMjZcwtziUkSLdqcNzgWkvQBYQd5jSwXznLCocj2RLATvJDwkJg3z1CsqWYBm7xVI7WQ7IuHJxHnCtq/YU0Xdd3gAKW3gUrDMEVwy9lS1nQchr6s0JGN5bwgRXCvSKpIKFBzYE5CcdhXMYKiUAUwjzQFBx52kPqdwgZEC5fqEMxScBzsNPaDYGDnpBNB8TAC1ogBXzd6JVLVPx9zS8HIJpAWFaCztjtnJj6Pe4lWvunmTV1lM88aBRjMAjB9zrSXbxn4gvJhH1dbEjefz/lDoNxioZRs7hRHGYRjTbwSTN1AvQ72UytwyQGzg6mBs+tgrH3cw3ERQkWrfC8ZIbrSzkn3lWLBpuLbHlKSl4VDViycAPhCginCyuD2mOIuBqRV72USOcKCoS5GguFKjbguzFbhJBkfqjERno+QEkHA+nGaaJCSQb5/21DwQCyNrkbCNAMwxIutzbijLtxBhmmZsq9QGgQn7njB7h7wnBE9AZeSyg9gjMYSDEmTslvmAQA6kXsF+PsPNMxwCmNTTAyPvFVAynfemKSPfL0CwJ6/1OQ1Wgy2sId2qGReosblaUDaPcF00mmttafNbgSMHkScVoyCkRvGChklKhMId0JX7f+0Z2FPitZDiWxpSlqrV1v01fd675m3zsCpuAd/R8+XBgt16Px/i+HiMiuGfGrU2Wj6ubxCDRaxcJ0+tjMpFs0FAcPoqw9Bskl35JWgLVv4m0Rx1Di5Yta9rBsbkH5kZ7J1ObHF/TWZShO40FcnyeM2M6opgPcqh2HfAOfNIvDRgyNUiMTGaXkhYwUfAOURVbqvMDexkyM4ZQsDpdI/j6SQatiHAqo3P53OTTTA8xwcTxhbGb60VVb8/eesESFcGjQhYDgv2bQUAbOuGzIySdvgpYktnOHKY4wJrcwqS7Ne+70j7jmmaxEMBWjP34CfkIs3STSdFeuiIvEGljFqDDgwLXg4PV7ombt+tBaAX9T3tQmg1SNZ/ia9p9a0imwSTgxkVnZCOCBhIi6aI38IK6vMHwFXI8z5g9905343XLTu2GyjW+aatTW+A31ucRa4BV4+N2b0PcXw0RkUtgNz8kcTGYzxolkcMTq1VFjVEcAfO/Bdg2Mpk0fNoXK5BM/tdSkXhBOs/bJO1ySO2a5XCRSIPuAo7LVeGJE1qMzZO+/8IUCjask5DKYL2qYkq1a2WMOcEhWQGd94BTvgwDAFg7w4neD+JRqsbcIKhjYVzTlirBgiSVD6TijpZfZLT+phSqi5GbhXTWTkjU4yYFwnF1ocHcK6gUhA9IUwRRNYvmbVLgO+hgFLumaG4mXg81qrEFjVXkRkIwVTkNOwgApOWD6BXbTdVOTV+3tk9dSCv94IHnA43i837piJXSgE7eaVX/EsG/3pRjkbk2473cUpuX/+rDM77wNnxufdhJM34Qe0z/YaHP3a8467dIN3mwpdagdZIysGph9JiD3gAPVMCmA80hkLXgy6Vxb5lFaQ4yw3v7O+pbMCuB1cHLso29QDIg520C2GuSvDKKsEoBYM5KZdAQx+TWWTW8ntlnhpDN1XlqXCF9w6n0wk1V1y2DXvOglmgM2/NqBjZDES6PgQIHuUKALQGZD54TdV2QhezGJg4z5inGafjCedvHvD49i34sGB+dhSTWyuoFpAL3TBDvEoHFj2ZWkWrJARQjIA2Y99TbjcmzlP7XNWXFhC89iJK8b664WS2olIghqi7dob1E7a5cJteraq6B8WczHiQzsWbW//O4r5dsFb/9D6VtraJoW9+t6+5BVct40S6EY2EutvDxqZ75+9e84c4PhqjcuvK6aMtJLjNzQuvgxFaY606FobKQR1XMBTG4s3x5gH9phGq4hwdTW83kXog5qyilqDuugCRUusjbUW3TTsZDqLOreCOCEXbagK6aCDNy+zwPqjQclS8yYoGGefHC0KMWOa5NT63NCdp068QI0otjTSXcxKjdcNPiTECJGLU4tl1cSivvJCUMkKMePHqFWrOOD8+4unhEbtWMgtmXUGOMQXNcujCCqT1zbVKm2b1DIiFBs+5KOgtdyptjOlwaPydWhKYZXwlrOnEN+g9aJ6E69osAEk4WxnIqYHnUhPUM0iSGu74hN0nYz5b8edV4engSdziG7/KkzBDRRg2uxsjMr7eytq8CkiN5uF9BqmdZ5zXH9hb+SiMiix2/bttDgY2DQPfgFdu1b+C8uub9WaZZRe0XNNrMDBXwNWecpM3WvqQS5EyeertN33worFhojxjn9vKQCVkJYPVWqA5StHisPALoztqUZy488JnkbBBmnpJBqbUCi5AYZaKZ+VgWOuOWgq2S0Gu1qhMtGW9j9jTrgxUNQp5x7ZnHJ0JaRMul0uvX9LwI6UMp6xVa0UKEKY4oYaAKUZs64q7549489VrnB/e4rIlEKQfkffyXQQEljDIsKB+bwCnG0OpBSDJSgmJDuBSpM+y8/AUxCCmHVQDoJKOJqFJGgb7YCB7H2tmgsiLBmXcFuQ8tleR2VTrwEEaDELOWSPGLsglt43bhxiOcY3vdANx630AWu1MdGWkbj2glhnSBdI2JVwbnduNsS0ofDgjcnt8FEbFfARwJxc1mH/IRLTh66guTLWeQWickv7CYTfQmyVxje54BOvMZzfSBKQJDuQDQohaQcuDzoakW3OWKl5pxA5QCKAiLrVT4NP4KuZtFS12Iy36M10PS1+7Rhd3iNPU08nocTm3sdHJxiys11qxnlcQSQr0/tk9Tnd32HMWL0QDkT2rNCVIvDkX4L1rdT0WkhAJj8Q7bWjuCIXFizjcP8N0POLx6Qlfvv4Gjh/x+PYJh2XBH/7hP497T+K9EQlOUWX8rAc0SlWxI5LGZLDdFmByyNsGR4R1WzHlBbFUTIdFlOiIeojKrBnAvqgt/DDjH0IA14g9SwHlLWuV0TNIUGPdvY8eXrmbz2C6ZquaARk93PG5K4MzzP/xfXZ+A7vdONfNwxnO/b6ixfeZk984TwXgXhis4QlxT6B1HsowMASYeI0ZklYUqJaaFKTUAEOARyLU0puB2w0cLb5Tb8H5KBkRu5GA9moWb2FZlrYQSL9Ayhngqp9TroyhZWLMWyESEFLaZQr64Cholqb3lyGSepbaQhL5HoKRSI1PncTI5aJEqyCPJ23aDkiqF0SY51k9kq5OBwDCNeNm2MZaEpFBlEFYTkc8e/ECn/3wd8C14s2XX+Ht2xX7peLhm9fIece/8C/+ZbjjUd7gBnwM1LZfYjEmNQsBDSSbg2i0ZDydz3AhIMQowWutjVks7+m6rTbIV/ibsxocArxHjBOA1CrCr1/fcSXC4JVoqXIXxBpAW+5z8n0ZmTZVB6/FPssueKwle6+3Yt6beT123VCvm0g31veDuWasfuOMigxI0R0Yg0cCMGrLgsgNr/AwWn7RBa4Nt2oBOY9lmTD5gFQKvANySaglY8u7cBdqVzwjgqiaOVvY2mDLF5CmfjvYK0CZ9wKgEolgUMkZqFkniHlPQmYz17VNKoYyawmFOxYS4wyA4FBU2xXgrcssdFwHoCQkulSKtEBVz2haFpxOd4DzLSOUcwExME9RjaoU/nnXPTfxntDG3oGR0968AMCKHAmpZJRSEZcJz148x+V8RggRh/kJ33z9DbaV8Ph0xs9++kv8we//HirtYHagKmAtgpdmWuxRM8O5oPotBFBVPRvC24cnrDnht3/wGQ53R8RpaqETHLXK5asw1KEtIueMrt8rhplI+6dzG1MrTxj1XW8B3TFrYwa2ex24Mi4yn98lsdnfAEQPiFnlPE0DxjWWs212YpAlRc5AL5dgu13GyHJoIu/NeI3FiNf4y/d9fBRGRUyHTGwmVtk/tF1DBk92f6pdoUu4KsJmdATMyyx8iG3HeT2DuQAkC9zA2R4Rjd4NWnhBFCWbpDF7j90ND5DJlvLeHhfNxaySkFrrUiW9KhOM27fxwbf0KcEpuNxxIIBRVL4xRgl9KpR9aZ6Z7toi8iwN24/HEw6nEwBCLox9T0NBHAOoqDlLxXTpWY7CpRXcmWEpyCKuBAFKC8SIixFwCFNASTvm44Lf+4v/HB5ev8Uvw0/BNePxseLpYcfrL98g/+AHCBPBsfQ6JlQwRRTnUYmQS8WP/+zP8KM//RGe39/jL/3lP8Dd8xO2lMC14tPPPsPdy+eoTiRBne7qTo2KHU49Oed8wyos1GwziSScdUEasNeS2/sqoXGR3i0mvRU/18808px6C46odVAYOSPj+24xloYTNs+1h3F234xSYDBJ8+hJUFvrn80YGoHy9Wd9aHzlIzEqchhW0GEzeRTNIkuVLhSci1HSo0Y/Tylh3TY4Bry5f3YW2zHIwdGItVBzb83QjGzL27i4TQoIuHm9Ywho7NruLh3tGMKelOSDk1IC0qKxoXMcMatBs88HgALvHKYpihJ/FR0VOIdSpQ/Q8XjCtCyNuOW4IucyVCQLAY9LQRn0RWQMxYPJKTUGqueKgNCAz6jsXC6CB227tsVAxbatYFQcjkfMhyecz09C1iMg7ZssdEfgkgVbIg92FakW/Mn/92f48x//BCUVfOOe8Kc/+gl+53c/B3tgPp0QZ20a7wTH8s431b8Ru2hgc5GQ53bxMpTlyw7FObBzAISfwur93baCbSHOOE/wLjbR6AhOrnMU7hrPMz5uBaPdCwagRuhWeb97uFcojOJ4o/EbsUT5/33X+SGOj8aoXMWcMLcQAFRYSb0SglDkpzlimlyrmSm5x8OSpx9Fksc0n95EA4BBrZzeOQfn4+B+Xk+OcZLYtTbcAaZZqj2O2aHW7i1IC1ThnghmISQ3aOr6cFiaTIFNmpIEIwEIaV+RS0FKu7QnJYd5nnH/7Dmcqeenot6XSCNIepqx7zu8J0zzrPwXatdeNfSpwyItVXLEs553vVyEPh/HhmmEKQT40xHeeVzePuJ0f0LKOzwBdyFKTVEG2GdwLiAHFKpAAc7rBftW8PLlJ8g543h3xOnZCYgBz57fYz4dcLw7ws+TGG9Hrf6mZb9uFmHRzFsDZ5lbHY+zsIk6tWAEZL3vDFgDeg2vMe3f21TxaBiE5nCd6bHC0nexEpXLUPxKNh4zWhYKkhqOfk3NUcVtzU83IUTqxfEguj5c04c4vkuD9t8D8A8AfAbZdP+Ymf8eEb0C8F8D+H1Ik/a/ysyvSa7+7wH4NwCcAfw1Zv7H3/WCmAXA0/8apmLWPMSAGCXc2fek4Q80nBC/wTd9DAAwwRr5uxl9Ek9DSFpdeWv0TIw81vkaNyI9BnA6AQKZNQuj5fs6gpLyBYMLwSlTNMaIeXmJnDfsKSHtOy7nJ8FhwIghwDtpPJ5LRlYDkFUyICwz7u6fiUwji6aJGC3rvSPgsSmfSX/kHTHOiFEWXFCjcX56EkB7igr2litwkAjw0cv4skgXCJgtXt7x7ojpd38H5dOMn/74J/h52jErVlJyAlfdwStJ90gmTNOCzz77DKVmkCcc7g6YDxPc5BDmGT6KMYHr4UF10ExZP0YRJVHHf3eXN7qAbQrOOVHel5uIcZGO97ZowaiFxiP13oxMI+k56bfcvCPuqebRawI6Qa5oyQXYSZtckgSA0+sSY6HiVFU7JNj1gW48lXENAc1HH0KyD3V8F08lA/ibzPyPiegewP9ORP8DgL8G4H9k5r9DRH8LwN8C8B8B+NcB/KH+/CsA/lP9/SuPEchsoKzGjRYSzVOE4LG5UbcxZHvMBRTDBKX7j1ZaJ47GwFKnEq48S5sI4//dyHTdDhNMFk6JKJhZpoAJQBW8I6UEOCnKOx2OcCQqZzllbJc32NOqRkCu1bg3tRSkzQhbApACQs+flwX3L1/h/v4eALDte/PMrD3qshwgSvJVZQ+05QZ3LRJr9FWK6pfoArKWpfu+CzFM7wEUg8gtFQuACNu2IXqPGhj3L1/g8vAAfryISHjUxu65AiHIXdJK6eNpAROjoOD+1R3iElvtkgtSwW0+PYMbl4ZvGKv2d6ni1TpyqHTNoCVDN5Wt7CztzB33eB+4Os5NA647/tXfV6uUXIyPjQbqWuri/U3H0Dwhe18XVTf4xb5vbfNZN0M7f5XX26ZGpOuAPiKjwsw/A/Az/fuBiP4JgB8C+CsA/khf9p8B+J8gRuWvAPgHLKP1vxDRCyL6XM/zK48WX1YzDpIqC95jmgJQs1Z02rUBAKsau96Mdk9Ybwr3kKDqTXCQWF1T0GYgZPKZknqfBL3bILXPHVW65HBAyXDqMVQASwi4v5dsTMkFKWVsl0dY6wmZnAxXtViMGQWiIpdKUvxAmlsdDgc453B3d49cGcE77NuGpJ5Fyb1LoQ8e+55g7Fxhz4qimrUCiTFiWRZpyQHgdHcHHzzO57P2PPaY49QWhNXY5CIp8L2o3GOtOCwHkPTyxLMXL7A/nfF2/0Jvg+yolaXBG0PqUiAvh58CjocD/OzhpwA4kSHw5BUrc/BhJJb1UNbuzXV44dqmAsW+au2LSpT8PXzVcGUoVRjnYUsf6w9sfqir24h2IzB808LUzgNcb1Yylr2dqwB51P60eTd6Qv1HvSbScTRQtlwzecew6UODtb8WpkJEvw/gXwbwvwL4bDAUP4eER4AYnB8Pb/uJPnZlVIjorwP46wBwd1oGF40xAPsIPiIEJ60itJ+tDOwwUK0vx2DlcT2Q7WY4B74qFhzEd1RPpKUkB8NSa0XRGDeGqG035aO99yAfwDshRMNOGDVvOJ/P2NY8VDOrYJAtjJYK7DtiJcCHSUK9aQKIsG87zucz9lTw4uUrSSnvmwDA1mdZjahoyTpM09LCHGZqxtAMS60m7lSxbRtclsWxLAu4StN104mN84ySpFNgyrkp+leWWpuSC6Z5xjTPeP7iJbbXD+C0obB4P0W/c1V8rDDDzwFh9gjLJGr6ivnUIu1hvbbpsO6HtZQm3m3H6BFYOGfdC9q9rxXkFRsLQQw6S9lAUjKid+7qfLZAr3A0DJ6FempmRJg7+D+mp8frBIwsh+vPcE5ieJ3/t4mBvmbs83v20wzN+45rY/QRGhUiugPw3wD4D5n57Y17yES/nn/FzH8M4I8B4Lc/ec79ZlloARFphrjtDqY5odW/REOcrOnhdq3QndNdGR+i7k6bVgqRTFw/iBIN13gFCJIXGQKbRFYNzHq+OE0IwePxYUdOK3LaADYVMnG1WxtOloVVR2yHQvMgGIR1u2DXEMX5gGfPX+J0dweAsK5b27lFJIjEXpWshX8LQogoxRTxBYwWhTl3VfQWgtewQ753KQWpJCzTLH/nglSkLYboyZBQ8oNUIqctaatSYIoTpmVBnGaUJBoylTUrozh5KjvYRxzmBcv9QcKe2US5Pba6KnPYUt2qou8casltUd6GEOYNGAB93dBc3s+1qrZsB95zzija0N7ws5H4d7sgG2aCPl7dw74Oi0aQvxs6ujIs8ggUF+xzr83Z4TCuYl8u9Z3zyZx8f5bqQxzfyagQUYQYlP+cmf9bffgXFtYQ0ecAvtDH/xzA7w1v/1197FuPq2QZS11P8F5UumpWKrvhINwAWaslgf3WnjRtEsnFg6B9ftn4AJLB8FqMiNpBNVPhIq3ekgWtvBVNGZqbG2MUgNM51MLY1h0p7SDohOcILg7spD5HamIyvJPFR96hemCeFrjQ5Q/O66oV0jImxqKVLoIbYOFXkLapdg3MDGpNzAqQCSmVNsrOVexJVrZJIlqtUSkFpMaFmTFNk3oNgrdIGQQ0PUtwPiDljFoZUQWUskprxnnG8f4eD+sqZkFBcjiv5Q4Vy2HBfDzChYjleMKWkxifyliWI6zlbC4FQYlsAFTigFq4Qc41IXHW1Wbs15RMxkKB0Zqu8BBSIL6WgozeUA1EiDH0uqXBAN+CtUaa5GpcK25e6C3mA6BthBiMjs1ZGu6JVVe35IANfiNX9LnRxtceZ2Pbynex0pAPdXyX7A8B+PsA/gkz/93hqX8E4N8B8Hf09383PP4fENF/BQFov/kueEr7PAfRgq0FUOanA7QE3nASaoCcEaHcGDcSpCUyVFCJqNdt6OtlUphotah7VRYejPNauaqhFjtGRhHdVJ2IMU5tV0y7hGZSTVuUGemBAFTn4WrCNAW4mZDS1jgNOWd4BJQC5FyQ8ipyjzeZBADwwYDYLBmQWlBqQDgc4aZZdvOsBYDMKLmiUurnUJCyKs6QaoEnSXGLgZxQuaiaPbWwx5HvqeZSpHQArOByRQiiLQNXEL1UI8/TDDfNKGGCqxVUK1wVqc/MFdE56bd8iIjzDDiPGJ3dZLBziH6WjpBg5CoZp9aJUt3SyiqpQMJOdiIUI56s68LZKSsJUtxgkTggIVHmoviI5JzF+KQkDcu8Q/Dhmi+Ca++l6vnEowIKCxboB2KcbQ4W3td6rXnbzgu5BjlM0VCuXRBY9cxglDh95dCetYVDDTOqqm917fF8n8d38VT+VQD/NoD/k4j+D33sP4YYk39IRP8egD8D8Ff1uf8ekk7+fyEp5X/3u12KuvJODQobtqLykQbS9SBHeSX9xpBiK0TfMoQ9+pEFyhImSfHf2AFOakzIuyaa7Lxr7S9q5VZPY6nSnKRYbYoRwXus64pSCEQFxWXJ9qS9tRFdlhnzPCOXgl31T0K4xnLSXsAApnmCbDZiABlAhfQmDsuMXGrb7aT1anffLfafYkRUzKGo1wRnZf2EeVlUW7c3/6q1NlHsZog1w8RVOCDeycILkB3WQoFpnsXr2VOb4KUUZGaEGZimCTFOWlTpME8RSWubbKd3kJICC1HiNF0VV14tdEB35SJgr/OaHJENRwS9JPwIIUimJgR4J7ovoKoSDh1YzbnoxuWuPBQ7GGjYjAIe+pGymZVa4JibzosZdyNKNm+HAGLfjaZ5GZD5Wa3oanRRhkzn+zg0Nt6jB/Shju+S/fmfgW81c//ae17PAP7Gr3MRMsRSW1JLBlDhqLawp+0MpCEKQT2O9qHtdQ3l79dz9VtAXXElbcJ475vW7XijahEx5hAjDvMM7yPWdYX0MDauRwRXRgwHzbIUPD4+Yt93mBYssZD5yBGWuGCaJuS84+n8JLR9JZONbr0jSUMTEXKVuigJVSRcmecj5sNB6kcqI+2biJqAsWdpEhY0fHEgpFqxb+YlaXhQWRZViELhFy1OrJcLHEkxZvBCeNvWVboGWlioO26pGQk7csqAC4hedHIpOIR5Qt61H7Oly2PAcjhKbZLep5xyI+X1ey0hCBX11LxvoZA3YFSd05KyhmiKsSmm0AwUQzElNCNbGah6v8iJZ5ZLhvOhYTFNc2ZYlONzJ1r9AAAgAElEQVTibNMP0CpsLQtwxoHR50X/tIc+MCNgZ+gbo9mMDsrK81wNxLXnach69jk7ArMf0pCMx0fDqG29WthUulR2kYCeGibtp6MecDMkgubLRsANCwHw3slAUC0W6l5BBy2VHKW4yLIsInZUKta0wnvf2mmIAagouaKkDSntyKpBQjqZnRqCoGnRnBLO5yc06QVm1WERN9zql/Y9o2TZ5UKc1aXXhR6jeCMa6+echIKfEqzDogNQdgl/su7SXj0Oy3IFDXG4ZlzOTxKKoPedWde1jdu+iWcWvAC68r0zkrYP8THidLpHOERJbU9C8yfvsKUE74BDvMfuGIfjUb1Mj+AVZ6m1YVYxRgVbe8MzuffCnTE1O8u8CKaldVaQliJm9Ll0z0XmhMylWlkqullrbNjmjRb4Oa8yn4yqnQvGudSMTZtT5omIhrJRHDqeYphHD5tsrFm/+2hUBB66LXDU5/QDx41zTGkD/blrgPjDHB+NUeEqrRdMPtCMh9Dsu2wkOandGUEzCVutQEsQ8dHlu/0b3B+73jG6PsbxcMC0zMg5Y1tXASpVEc14LQ0Q3KXPsCcPigaeyQQxhTJm4HK5CMNUlcdqLahV1Nbu759jnhekvOPt2wcQPOJ8aLU5ln0CgFwYXDKcJ+z7jn3fgMoScTNhWhZMIQohTyfZvm6ye/sgejAA0rZiV9ZojBN4V2nHfVdPSwFb5xB8BJFDzgJ+3p3u5DVVgtJSKvY9AbhgmWbhUOj9ylwxTwsoyEKtxEg5wbOo1s3zjG3fJN0b+5TMKSHOk3p2/btYaGcArHmbFpKCIUptxmGpaJouEqJJMWaME0rJqDUp+Q/IyWqmRFTK3RoSoIPENn8AWD8nIZDoIia6Ck3IdxFtO48kAuR+XFc/j6Bt91zaj4Z2gBWw1qs5zUMngQ/tsXw0RkWmpvy0UAbmxml1rHaia82clP4O9J3ApAsc9bBnHNyRXg2IMWKSXYeZEVj0X4kIl/OlUd5ZwzPAYmwFP1k8GsciSl0VKJUQIPb+PPuuE1jOJyEOcDgsWJYj9rTj9ddPgBMKuzT3QvfEvNU5lea+U5G6nnmaMIWImgu2bUXeE9bzBeu6SpxPUsQm2aqKnJ0WQwpG5V3AFGc4DXVM/FrG3iGECVC8Yd8zliUg54JpWpCzcEcAydQQEdZNDFicZ9TLJiQ+77DuOw6He1y2DW6ahQejOEmMgql4XbDGoRl36tvHxnDJGshXvsY95H2MXBLAgqeIaFOvYE6poNTUKAbek7YwqSg3qVmbU0A3Ag4iRMUq3ekUuBWOpWvjomo/YAjwPuJnpJbDDEi799DQmUwCQ70SzQ7ZWNi4tLIBIvDA1/noUsof5uCGyDtzK5klGzTU3ijkpTtUB7zsTpCCu+Z6ymMd1JMcPlqKTcrj5e/D4YB5jnj79i2AXvtjhCoQI0SZMFzqYJiAXdt9koMAokVAzpS2wS3tk/F4POL+/oTL+RGvX3+tivYRcZbuhD54TMusIthSfCiC1k5bkuqE4ortcsbDnlCLVWJLHB8n4X44Ry3r453HPM+AylXmUuBDbJKZMQRM04R19dg2IddJsaYs4BjFAyrOaY0T4DRDAhC2dVMPokhlsJMQsjIQnMN8WPB4WbGcCpwCweu64nR3QtCuA9Y4XpT20VLDzkn2yrfOhYOrzwywNLuvurhzNg9XyjEsFQ+2yhiGiwGRJ+yPG7aS4OG0uLDAOa0u13Darss6SALGUXEtFGNlRIsUQqdCmESBYVp2mGG0Pt02S0ZvhsgyWbXhPAyAfF8X9lq7JpvTY6bqQx0fiVFhkAr0OO+0oEomKTnVzyBuOETHZ/vAtZhWz2cM1r7jmquqBoctLSoCSXf398g54/Xr142sZM29hJhGg3G6FsQp2dxdAAzkIv2KzdOy52xRHk9HcC345Re/AOcs7TKg8TyzAKzGh3GMPe3wMWKaJp3cG9bLKpwYzYwty0F0ZCu31qZxniTzQJK+BgPRBzAc9l1Ci2WaEaYJzJBslNL+K3PzIjw5lMLtHEFreIoahMPpCFQBc6c4YVsFeypVdvqnywV3pyMOpyMKV0zzjMenJzyPEVQKEjPWi0ecJ7Dp3NaKPSUcongW27Y1DKgkKRGggR4vhjyrcYF2gRRpg8KACwJwV+W2AFLx7AGEacbxjrCezwp4V3gSD8E5f9W7GQAeHh40YSBkukLWN0kJdi0k0iyW70p6HSfsXhVdZXWu53atoq8jRntg5RK1JnS3a0EMmoo73XhuH+L4SIyKeiheXE82kM7s9ntCwtEt9s619qeseInTXW58rd7N1sWPnMPhcECcZjw+PipxrR+Ng0CqIztkksYUo2UdghPxIm5gY4ZR9RxXzIcjgpP6mnU9i6utoKRp43IVvGGehVwWtVk6qWFILQRihOgxxYjj4QhyASkX1LzL7mjuMEt5vQ+haXBwrZjmCURKagPawhVjljW1L5MymWg1yT0qtWLdtgbQbntCVsDWeB0hBGzrBdsqwKrzsjgdy8ZhfZanaRISHTMWHcfYgHBpRjZZKvnG4IYQmodq2AQ0JCmlNO0VrxhXp/lbJoWROcORh/cRx+MRyXuktIJbHU+v8zLMZpompGRdJxWcpQJfPUgFuA10tUybQMSMrNk48h5Qg+qZmxaLdEQaeSdjuUD3SuqNFRq9cWZu92587kMdH4VRIUi1rycV8ykV4CLiSb0YHGOxdxdSAqCd46h5J303gGE0qhYppeMSIpyOR3jn8PDmazgCIlVAVeTA1IA6x8pHYEaSFoOoWYrLBMQkVPOSvGA/tVZQVU/HeRyWe+zbioeHBzE63jdZS2GFisYKO2rM02d3d1cLyLJF0Xv4acayLHBEuFzOUOlo+Egg6g3ewZJBnyapz0GF6LhkEbmqlbFvFaiXhr8E7yG7PISAGCbsKcERcDk/oWoYMU0Tzk9neE0/x8MRJWd4LyUGMQY85gt2ZETKyDUhVMFuvLJw056x7RvSvuNZrZjniJwTpmkGCEiVOxAKNM/RskJSUiSC3J6kAyOYAWcUfSkrkBKPCkJRSU2GMiRb+OxcQJgknMtpR067Cmq92y/IjLxJUToFv8EyL2zucGVQcI3nwwRZ8LWIWJSzUMuSEFZ20ss/CELuM403y1e/D1uyPbh+QCNye3wURgUQw1KVRWsuoVMBIzMShsSPqmzmrcg5rnPzFrqaoSm1pwXv7u7ApeLNm9cilajpSyNXGWhnOHCtFVx2Sc8CUkmr6UALq2wnTTkLJZ8ls+Gdw/n8hH1bdXJZ5awCc96wgoBnz15oytXj4VFSzzHG1nfmdDohxijGYN9VtiAMIDTAWu+z7ZtUdTuHCwxXlh3d0rZw4kWkfcccpSXqtq43ntgFIaqSvYG7XjCXZVlQU1bvZ24Kcjll1C1hXTdc1hXPnj9DVg/H114ItxwOknXJCU9Pj0gpYlqW5nH6KWK9rI1uDxIQNWhpAvR+OOcabiElGIPGrGFaXJA0hAbLApbWHT1ElV+uMaaz1l6xYTyGyQFajFm6tGXD9XqGEbq5EROiD1efYSG2D6Gdw6qr1ZdqYHnzTAa6hB1XnBQN+YlqKzv40CHQx2FUCIKZaObEjIPD0LxpeDnbTtBkF9XY+KEaaMA/qtWIq0t9f3+vokhC8urpONeLxLQiGobDKIvUen2VtKNmeY4UwwHQeRUAjscDcso4X56EOAZ0bgREfMl6/r765BPc3T/Dngtev36Ny7ohTpPUoBTCPC84HA4oteLp6dyMiKWsLWOSUwLnou65TKqi5QQ+BMnwuI4zOSKcn55kIpeCnDOmabrqCRTjBJBca0rS6uOiFcxcK1LaW/iUc8a2iZzCw9tHXC67AI2Co2JdVzybF8FkSlbgMyJqKtl5j5IrQugiUhZ6CB+lqHg5I0yxzQnDSlrvJu+7Ypv3AApKGrARLti06lp6Svesi/GizDvc9912JjjnMClfpqjejKnmtcXNVhukhhxVykiMug80IFU2mA6EV/XMrI2tZX/aOkHfwDpGiH7x+mc1APgDc1SAj8WoQLIVXR4AbejVBwTQ3UKTE7QbYem2Mczs1tnIcnK+490dUtqxJ+EmVJVrBIsINMqQJVLXmRRPIWE1tZ2RSy8uNBc5VwENpxjx8PAgSm7Ky5C6HHFNLaNx//wFXrx4iVIrfvLTn2FdV0zTjOPdHV6+eoHDIrIQDw8PePv4jYZ9QWnYGayygfu+S9yeEjiVZmSICNU5nI5HSfECuKwXSUXPMxhJZAumSeQPnMPT01Pz2gArtCMBJL2TjJLyWUrOKNoTWQBMj1pENuFy2ZBSQYxBdG2DB7xoCTOAaV5EuyX2bgYCwHYPtJQCsveoQSiloG4VpYrcQklJF7yHj+FqV27paCa4IIQ9CaGkoVzKK3JJg4yoTIdaCkphJRsGCR2BlnWz8clq7IBOGhTSXW38EbbwRC3XKKlh8pGSxtdZzzr/G4IrKXmHa4b5VXbTvu/N9x7TzB/q+GiMSlXwkbhBJGIM4OAVWPWqOWGsFIk3m9yRGJ12xl7NatbmdDphT3vrNdxTzL34S+jdXvkESj6ysMpRC3eMXNRuGmTHORwO4Frx1VdfNcMkaeWqBYsEHyZ88tuvdMes+PkXv8TlcsG8LPj8888xzwvmecK2XfDFL95g3Tbc3z/D6XTCtu8ouaePpWygXim5VRJPC47w7NlzHA4LLpcLzpczds3gMET8yoeIg4o1jWCkNUc3g8JVdFp88MhZDGfOGQ5AXBZs24aUEqbo8PD2EefzGU9vH8VDI8LlvEp18knEplIWFTpywLZtYuB0LCeTktQ5YMCvLdIQQgv9zEMkInCQ6uzQGtJDRZms+ly8tJISai0oVTsQOsmU5CxFhCbikdUATcus97fAsUNhMWh72pvxsPk0Cnq1hcwVzBqqEbXs0BVfikh5/2qUNDxuxMxawdTrokZgVj6CO1WCcfWcPPybZlTY0rtGDVIavbMqZNfcSmtn77zl+8WgjFiKxKrXiPesTb9NKrG5hdSBOOdEE5XZMBVq2ApXRs2pXYt5KKI3kkAMPH/+XHfoS7sOIb/J+06nE549fwGQw5s3b8Asz59ORzx79gzOObx48QJvv3mDX/7ip9jThtPpDs/v7+Gcw9u3b8UgomNLh8MRJRecLxftlyz1Ss9eCjZzOZ/x5euvUZkRfMAcp6EGBdqsXclSIMzKwZhixOPjo40oLumCeZ6xXi44nu6FTh8C9m0Dg3A+n6WU4bLh8eEB67pJ69WiDcNXwMeAfdtlAxnc8hBCY8DmLHVAcZLFZGzTKcar3Xme53Y/7btEJypxUsckeIe3xU1tK5IwEAHrVpFLAlFFCFMjNJYiZEW7z/u6aWZJ5ksIHo4WcKl4XB8ES1Pd4d6kzqqHpZSDaxGGMUTzhpQ/Q3BSicxQT0QBe7ZwWg1L82K6J23z2LJGMCxnyAL1JfabZlQgcSwB6uk5vXHULDMzo+ZOyZZ+OCPVftRccW2HI0IjK+37RWqMzCshCY2Cj6Lm5gi5itExMSOuUm1acoGD4D4li6xlKVJ57LzH82fP8HQ+yyR3Hafx3uN4eo44zfA+4OvXr7HvCa9evWr4wePjW8zzjLvTCT/60z+RTMize7x6fo89Zbz+8peSaPQB5D2m2NuhPrz9RheW0Lnvnz3DJ69eSTj14x+LHqvybMx19kr+mqdF0rzmjus15yxV1fsmWEkuVQS2a0XwAetlbbVR67rBk8PlssGRw7ZuLVSMIYCd1uQoeOuDw8PDg4QttSBCJrzIQJAWKUr9DnMV6r6GYd4L/pE15T370GqPaq3Ytx0LzeKNqGGxkKOC20YF7Ul98uLxXC5PWnMkhsXCvm3bEOPUvLjW0VJ/z/MMR9RS8Yb/MEuzN+G5EFrpYSmS3UNvIu+9tB0RUJZU8d8yDEZf0Mc1lAc6JtNSzNqWFkCjV5gn9BtZ+2NGVMSZRvYsZELUChE+umYQ2mKQc/TiMsNSnPMIwbVMBSDpO3utsE2lrsWyB1Fp3NZuo1IVkpNiISi1JbYrCIfjHQ6HBev5IhIETmprzF0PwYOp4HxZsW0b7u7u8fLlSzw9PeHNm29wPB5F6a0UvH3zNQ7zhLvDgsv5CW++egKzTOIKwjQTnt0/Q+WKp6dH7HsSIex5hnNdNe6rL7/E09OTMFShqXZVoncgEYRyDvu+oWoolFLCFGJbIDYG27a1RQqY3i0A7/D4+CjGObMSy2TyLsskeFTxWmApwOe2b4iL1iRNEWnfcbo7Yd93qT1iASgJ1AoHHRGCcy3TRAB8tLSxeD/kHdKeRGJz2zBNc+v+GHSCuSYhagWkotszTQcwV6zbWdqLOJGbnKYJOVWcz2dM2pqkqJcLKHahntNtOxBm1eMdKPLNw2D1kqlnMWGMZ+ek95N5OcOGiSGbZeqpTVfFkXg8tSo80D33W3Lchzg+CqMCqEEJHkFvvgUzAmT1tLC5fQ7UOtCJYQjNiptxsR9bKOYGxxDbeYhMAc7U3WzSMaRPceg1N0QAGZvR4bgsWA4HPDw+ggk43d8h+IiUpeXG4XhCrRkPb9/i7u4On372Gb744gt8+ZMvMU0T/sJf+D0wA4+Pj5jmiKe3F7x5fIuUNq1wll2r1IpnL15hORzw5vUb0aD1wPF4wvF4h6enJyyzFB5+/eVXqMm6JWrRmzNkigHtl7ytqxYNdsbnRWuFmKFArnh4wUt4Uqp2B4CDYwlV5nkBPHBYJuyrqOp7climGZfLGWUOSFsSJf/KWC8XwM1wwaEo7Vw8mYIQJ0zzJHiFlj3MwaPkjMNBpCWi710IZdHJBhHnCXVLiCFKFpHQ8BwKAYUYNUuVu31HE5+ephkpbw1IJyfLYpon1JLEYJHU19j1joCohWVG0BOPzmmGqyjg3CvtATUIOu7OqUZtrVKHBt+A2p79FONjOzDLh+saGWQkNR1NgxGh4Twf4vgojAoRw3vSIiuCIxMd0hSakw6EYwGWof0harNy1YIlOFj7S6+d9xiC/HZPpw6GS1OyqkRvbSAaTwVoNR/Q3WSaRGAphIB123A4HOA1Nb2uK0IIePXqE5wv4r28+q1PUUrBT37yM5SS8elvfYYXz1+AueKbb75GTRt+/ouvW80TmcIXAYe7OxyPR2zrjp///KeIccLxeIILghF99cuv8MknrxBCwFe//EIZw13NnVFRKpBLFg3YMWsA5XdY+w8ljG3brvU/Umu07ztyLo3nsixHcCXcnZ7hcDgqcN3rTaYYcTmfwQ7glLC7gBgz9iTgas4V+7qDcsF6vuB4OokhiMKgPRyE1JdLQd52WaxxAmpF2lZMhxmOAhwcqhM9Fu8dwrKgZuE6QUq1kNKOWovME2kQBUeSpUu1aI0SYZ6OWo0uoYQPHjE44LhI14IibOYYIrwa2bRvuq7FE2xGHF2xjVVjucmZ1iFMgXqPqh9MrhM9AVVr0xAdXId0NJrxsPsMmOdOqmao5wc1cPhDHR+FUQE0E+FJ2YhyiNdHSrrClbX13gM+AtAKVDCcnwAI4OhDgFVLOO+F7KTZIWlB2rM3wmQMrRYEgLrL4v4nxSxCnASDCbGBiyFGadSlwN6z5881bVkwa6Ov8yq1OnGa8er0CnOIeHp4wtu3b7BtbxX0dGJFIIs8TBEvf+sV1nXFF19+CS4Vz549a2JFl+0JBIcffPopLusF37x+DWlvKulHIlJ1fdkVD8eDKE7m0pX0tcbGG2EKhD1n4YJw1ZBRx0MxhRBErQ3k4NWwEUlWaz7OSvpjhMMklP6zeFsxBvhoHfuAkis8COeHJ5FpqBWH46ER2Iz3gloRjkfsKvS9nVcABfN8BHkFJl0HNil4eDhRgAPDsRRjMkuYAmv7CmjtTYXngOhnFF+1tKLjI+SkbMCrklqpLG11gzReS/uKRtaEGA3WVoKmbUwKEjsywXRuxZJONYLJO8VTBq+CjREMzegMxqO9pIdWTr3SBoIrHYOcG0zV9398HEaFhLgGs75NFtIyKKVRs+2nlIKcCkCiKWspUAFHHYJqwAbvYCr2VclWQDdcwrRU46IhlYkwiSRkUWFp6bUy1oEkO59zWI4HLPOC89MTDocDtnXD4+Mj4iRZi/v7OxzmBdE7fPP6NR7evkWtWb0wdY9JJsSLVy9xOCz4+Re/QC3SV+dwOODx8VHboDosywE/+Pxz/PKLL7Ctl8bhsdBg5Cbc3d21auAQYnPVm24IOZiKWzcoAoDGKA2+BMTU94AxTdKiNcZedIkinQxzzoIpkQN7D1pXMFfEKor763qBsVTXywWP3uNwd8K+CSAqEgIeh8MB6ypYlC2cwkVIiy6CQeAiNVBCf+96vCYoSs6h5oKSMkqS7zNN2kq1mFKbtOgQAh6jlLHLowDGpOFP1zchLZNAC69N0U0i9qG8wDwkKy8oRb2PnsWRQsSuRtgSCTZHbaFgaCZGfaO94qo0HNJY6P43z1MhjY1BZEqPjebMrOJMQ8wo4YgwFP0AjIYQpVWo1tU4mBJ6aTyYlg0wvotmaJyXiel14WzbBiLfsiyyXkUxbFe2KjmHWUWMCBJiLMuCs2aBplk8p1cvX0rGKCdhmT4+SvU1CIWDXAcLk/X+/g7bvuGnP/tzzPOM6XgEV0j1tJcCupcvX+Du7h4/+fGf6Q6l2TMtKzAeTggBi3JILpeLgsdS0p+zqYUNfYM1RcuQBXo4HBBixLIcYH2ZzVOaZ8mUlFI1nSpTycoKGAzvJoTFNY3WnJJU9RYpYyi1IKeEy+WiBaAEHySk3dMZx9NJDG7OSHqf0r5L1sivYjicQ0qyoKWmSFFMEUEAIONggt5Jr2FeFgVaZVNJaW8bBgCUoTmXYSmyoXmwyYOSx7IcQeSwqaEELAxRwFQ7JPggnRuIHOZZexcxS4O7AS+8okbI4gBghmWg47MxxAeejuEpNKj+23XQh/NVPgqjArSxuxqkRmijTjm2Hx+C6IB4MShRKehg69tbYH2JTaayl4L36mZnNSXWmoNIG5oLgNd6y4CVv9D78MQQVLejCwRt+y6yAZqJcc7hm7dvBCPQnslNGR9QNf2Cl69eKRflDS6XC+7u7nQR7KhFQq4K4AeffYaUEn725z+BsdiMJi6FhzKZ7PPXdcX5fGl0fjE4GaaiLyC0LLKUEwokkyONwx2WZcE0RdRqRl1kASpXrJcVp9MdUspScsC1YTchBKR9g5smuBhQUsa0LNjXFYfDodHut20Tj2VdkXLGNGnfI67YVM7SMis2D0xyoTJr4aFkQCaS0Fe8SoY1aydPCGSlCl64NUMqvSoXpjOpxzalMi97D6EK71WGs7CwkeMs1eVp6yr5FsY3ThQ1JTrZCNTYvmetj9dh3pd99liWclXzY++VbzKsI4cB2v0gx0dhVN4Xijjv32ljMOp6xhAQ5oig9GojLlk/YVIR6FpLF0ki7eFDofFZvAGzuohKKar0HsFMnSzH0qHQYmRLbeecUVk0PiymncOM0/GI8/mMr776SmqASkZK6t6jd6kDHF69Euzkm29eg7niTquT076L5AIYMQb89qef4c2bN3j79i1i6O0+jbruvRc8IzhMU8TT0xMenx5116LGt5CCRNGSMVnEkjNSyZjnQ8NNpnkSecuHtzgud03CMEQpLDyejrhczqgVClx7zPOkXtGMdZXq3eP9HZ4eHhG9AOchRvEWS2kGhoD/v72vC7Vtycr7RlXNudbe+5zbP1FMR5uowTyYF21EBMXHGPul45t5iEIC5kFBIXnQ+CKIDwnRQCAIioIJJhLQEAkJRIMQAlGj0mq3TWsnCrHpqK237z3n7L3mnFU18vCNUVVznXNuX+GevXe8u+Dcs+8+a+9Vq2bVqDG+8Y1vYF0WVNOiCSng2dM32Zp12/De97+PONVhNkNENu31s2cQAebjhLUUHA+CMggCNi2c1Lf6NE9YDKNhg7SCqhHA3LyVkYmtuq+z8qriELs27jwfeGnVoYEYPBwl6LxtK0Np/lJsJZv3wufmRZDNWNBVwVhu0jgnxnfxs7Ejug3YCufkwtm3M+6FUQEcPKVRSeY9EICr8I56AFrjrGmeERPDklIKts1ceJMAoAJYNTq4ST7GAG1WvMeyIaWWufDfz9Rhl+mDkpQngSFGraxC1jqQmGpt4dIbb76Jm+trPH78mIV464ZgkgYSEmrecHFxicPhiKdPn2JZTo0bkjPDO5GIXCseXV3h6uoKf/rZz+L62bOmHyMWf+ficgC8cadDwvXNMzx96galsB2JGejOQpXh5mO4E+NkHuBkmikrri6vkNJMIDuxpWtMEcu6MsyJ5MqM2JfjLDlvfJ8YUEURZzbpmgBcP33K9UrUjlEjsV1fX2Oa6cHlmAnYqlpBY8XV5RU2w0gg1Cgp1zywMbDyuFi26nA40IhqZf2MCKZIWgCBaLXDWgzQ7Z0SAA9L2OmSl1NtF483M67K708WXnb6glEiAAsVCzaQFJhiQhIgV+5VVGt8j+5TiLHJ/edHI+KeccNf/CDZvMmw5bmqytKN2xr3wqgohHEqdNjkrmq+Z6e6nmmKVBajJQYzEsMo5qGMIFgVaSnqnZK+1hYGjb1Somm2AnuiXamkmTsukwJvrAtLjb7xxhuoubQQZllOnG8BQkioZcN73/N+QBRvvPG6AaIzADVuhfViFsX73/d+CIA/+eyfsBVFlOaVCaw613RkSymkrxcKII2ushsRx6r8/90whRAgMeLy8hJpOmLbMiad8OjRBUJkwzPq5vbShGAhoiDgcDi0QkTHcRgy0GWnd0M9YBHBcSam4aCuVxSrskVIzqQFpJQAS3OHGBrb1zODuQ4eaebPksJPvAiWFQkpwfVdQ4yYQ1eN83Ch/39/3mNtDkNwxbIsZNsOBZfOCHc2uHObgiUMPPGgNk8XypZII5ASsbWtEP9b17UZFYbgCm9dSxAXjY4PoBkf56h4Z0V6KwZdWUwAACAASURBVO9GTEWdjBQQRezgdKq9ewLzPLcG6rVWVhVDkCIp8KwI7thJCBEIe/dQYkD0B6NqanOp4QFu8keVLX+tCJtVecw9PtTD8YhSCt743OcAAMerC0QE3NzcsDweaD93+dpryOuGJ298juzNaQKU4GAwtfdpmvHofe/F6foab37uz9ilMZiYMidlamHA4XDEaVlxcXEBkYAnT99sAOOYBRLh7UevSnefj83gj+ZdlHZjL8sClQUXl49xsEzWzangcGDz95IrjoejsWKnJhfgVdKqVMGLMVqRIzDNMzRXhCvB9c1NC0NSyfbexWqKDHScEm5ONBYlF6zianABpfrBZ8GdezoxErjPOeP6+hrHq0u2hAWTizFSA5hyDRuCZQfGPRdMOMthCz/c5CdRzjPZXnRcbZ6o9+saUMVahlQv9QgBVYFVF8QpGd+oN2ZzpUH3dLyivVZeNO2iEGltPQAYT2gPIQCgeFTt2NBtjPthVAC2TwjBLDEXoVbTVTWMw28KVROrNn4F0LU52BF0kI8U3wyUNvCet2maMJtbvOXcXusZIRHrMNd+j1j83OnVDowS3F1wurmBCPVHDvMR10+f2Q3C8CnNEcf5gGU5Mb1rOh7VWcOBqd2rR1e4vLrE62+8jpvra96alfwOuM6GCmql8v6ykOae0oQ3n7zRwGuvOO7sYXPlIU7wBoDGzlQAy7ZAJGFKZAZLoHG6ODK8YUUx32vdVsyHCfNhxul0snKBwPYbKSJY+45pmrBuG1AqppmpaQ0CTMARHZw/AIjbxpYjQZCbCFU03V2XhTQKforYMsW9mCEMLeQ6na7ZlSCxvmpdFhzCkZ9XWXqRYkCJ1rO75MahcaC61fM4H8ZhDlWyjY3v5P2N8roh1woUGQBe88BtfbUWaDS8I2dE8yIKFKje1M69SjZ7m9LULjWvqFa1FDo8nWwedZskWohkk34Fh/bF494YFa6F3wpcOGhACr0x1LYV2OUBTxVGc8lb+Xk1+nXat6psFGwILg4sFlxOJ7Y7jSwmLAbGpUSlNYmCaB0IS61sTamULazWR7jWimW9wbqcLIwh29aJca7GHpMizQnLs2vqlKZI4mcU1FxIz1bB48dXCCHiT1//M5T1GabI1hjsOxSsVCACQs/NU+SXBgzHkBCS7Nz4Vq8ighTnJpHgzM6QWJ7ghLHDIUFCgRiz+ThNCGDtjkBb5a8LcRdlz6Y0Md2sosZXqZhiQkgRCcpU8ZoRJaJG8jCidE8QQSi8lKxaOUSoZuTVQykenWK4xbqsrUUrM3Y0ygAxkuvrJ7i8vIRIRFgXbKJWNxRQ7bOTxxQbq9gzTU4wqyAfyrmuzq8plbiS67cEVcQ4tXApgdXI27ohTV0XF8osIkJArICioCoxLo0KaIQaK9zxO9Qx5eyZqf5MGQ55s7Eu6kQpkYqm5XBL494YFVaQdi9BFZAA3lgNbHWLHRqgCvT+K60uKARI0HbTuGFxPEarUtoxdVfTvZuUZqwr053OiM1lMyIRmjZHFBbkrctCb2fdME0zrq6u2IPXskosVRHDVm4ggYWM3lO5GBEqhoSrq0vUSolLqBXp1WKpUeyMgzONnz69MXIbWaPzPGHNa8OgRq8KQLvlGqbiWJFyEx6Px1ZnQ6lEennXz57htCwNIHfRX0/1j6X/h5n4zlYroolW+7MBxGjp0sKevgec05GwrRtu6hPUDDBLVRGV3hXnlnFzkzFVA2INj9m2DLH2uKUUXF9fs8xhZSnGMbDkg7gMPYYYvMuhUevt8okpGgO72MVCgxJSRDDhKDFwHaAXnUIwSn5FOEzY1sSsnwSEOBlwzXPOfVvQigEb1kVvOgozoJb76R6IjV06Wf17XK/zAsLz1POrHJ8XvRGRD4rIL4nI74jIx0Xku+37PyAinxaRj9qfDw8/830i8ikR+aSIfNPbmkhgsy7K6HUFuNbfxoGtMACLjouErvfpxmaHo4g0PCbnzIboFmo5QQjoKukufVAK6zagYi6x1w8RfF1Mw2Q9LZimA66uHrW5dVp7akWJIQTEKbWwjVXUE6Zpwnve+16UUvD06RNz5Y3LgP5Z++dmVuPm5nogoWWEIAxZhtcCaPgGK7Fd9SxYJkfaHH2NWLrABxBDwLKc8OTJmw0fcVq6r/1o1MfN7JiOfy1C/GgMx2IkwZB8GNdymcnYDcF0Tgzzypn9rY2DVGvBzc0znE7XACq27YQYOwDrNITr62sz6ivWZQEUJnu50PPJGcnXq5KUqKWgboWFebUgbxTHKiVjWxbUnDGlyLC0ZLKtBXxusBAqshPC8eICEiNCSiaBYRQIoGEnkIHgxpwzVJlh9Aj8fPgeHy/P8RI9J8bd1ng7nkoG8A9V9TdE5DGAXxeRX7B/++eq+s/GF4vIVwL4VgB/A8BfAfCLIvLXVbW87A1E2LGNG7UYB9A5AnxNGFi1fjueVyN3Tste8q+niJ2OT2MlylAJyopUur09PVoKbwuxGDhab92nT5400lY1Gv98cWHGw0lVoSH8q/FTYDdSiAmC0jyGi4sjnjx50ngspeQmTuRzdN1UqFpV7YZcCw7T0Tg0pQX9o1EduSyOs4hQI1ea8ePaTZHpXi3EIFKKQFUDGLsnSO2TvmYi0oyNhyBu/Dr25B30tFU/n04nbkIzup41EqFWyeXlFdbTCSFQf8SbqVc1qQbziDzTRQpAn6sbtFoVNzc3mKaJ+JOl/dd1JfDtnpyHPiLWXVIhlfsk54xq8g/TNGMzuQYHvcuW4czqZiggiNOMBGCajyYDkVE1QKDI2YxBLXC1waoF0eYEqcSepAKy3/9+Lvz5VtTG3n2REblNT+XzGhVV/QyAz9jXT0TkEwC++C1+5CMAfkZVFwC/LyKfAvC1AP7Hy37AQx6/8TBYWKZLmXKUwXC46yyGubi7y5ha2s0IoBkU+wytfUFQtlM4HtnTlxmmYxP9abep3ZiqGU/efAPLzYmVr5aR8qbpqmgyAu7ir97fNwgxGWFtkkqPm29Op2YEVQMCIhuLS+StVYmpEJBjvcpqh6ppxZwZ3DEVPnosHop0kSsCyTH2atgts0+zCN+HGIYiCWAlmphMY8QJXTEKaqW3tK5Lew+Rrtsq4tXf9EopVE5cYZ4pvvTo8RVB7Dixk2Kt7DZQmcZuTdtrQbCq4LJlZN0gIZqcA5nUAOw5RuQtG11/w3I64XA8IobQAOYYA04nE1syVFYg1K+1hnYn6wu1lQ1TnCxL18PwUaM2BEGE7VnQxsxGWSh5ZXgU3Ktw48DV6Aah4zgODbTvn3njUIWKJTAMW+FgSvm+eSptiMiXAvhqAL8C4OsBfJeIfBuAXwO9mddBg/PLw4/9Id7aCBmZzGUcubC+cM2YxNBizzRPCClaSBJ2txKMoBUE5m1Y0d9IWANvx5gS4/9ayCCdZ9zcnPbuudDjKHnD9bMnuLm5gSjTuDEGxBAxHw5AjMMH4jZy2cZ22EFOCx8wOQen0wlbYfbJMw4hWiZoY7l70Wz1PYppmttndUlFxzS85qeVFminm3t9z87I5ELJCSPOQYB1oxK+10B5qcMYxsXE9XFPA+hN08dbcvQge2rbwxPzasyrdAJZStH0bkLrSEgvJUNFzYsLBtxTWKpaOrWUDTlRV6eUbTh49A5LzoiTdTt0Xoh5D8fjASHQUyaGk9vPNzGs4wXW0wJUGpaUErbhc/pzAICy5WYsxvA1xoQYBaVsFFJvqWtSCqDkPzGdbtnQgfjWMJch5GyHyP+Ii7Z34/QcIPMKx9s2KiLyCMDPAvgeVX1TRH4UwA+Cn+QHAfwwgL/35/h93wHgOwDgtUdHaAWkIdiexe2LygViSjJZK0zXMHGquWMhtVbkden1IuiaKOwZTIOSUmr9dohReHccobhzjJAk2LaFGqwbpRPTnIxMFZDmGXGeMbKgfTPyIFszdzggXMxNBZblBAgp+Ox5NBzAWiGxoKfMCyQoUppwc3ONeU52u94gJS+eq5imA0YGMkCD4uGJCzzXyrR6Mo+DAt1lh400Q2EgoQhlBC4fXXVvzoxuzaXV07Tq3+jYUegZFa2IE+n/MUbUrbZsjt/0aaKK3Hjzb1kBpIbl+IHJebN2Klyndd0wTb3GZhTMptBRoZKctRgBgJoLpSmmxEZq2kGMGARAwrJtOF4cCSIvixW+KiBW3a1W8Jir0R8U0IyyVVTrEMl9TXr/FI/WiEx7UebMQthqrF3fSxLHsg73aLALdUTQeD274fO8b0ZFRCbQoPy0qv4cAKjqHw3//uMA/qP976cBfHD48S+x7+2Gqv4YgB8DgL/8he8h3u6tNvxrQ6g8NJmmGWmeWtl+L0+X3W3hlaj2Pu0BNRDX8AX/2XkmG9QzMQ5ehkD3/tmzZ3ZgqJHqeEdMCXGemebVczlL2HwAoNhncvSZN2FKCWGYn5doN/skrnnahZQ9a0TjWQaDQckGvn6Pq4wbz9Xxvc2Gu9s5bw04dq/CwWUVbVki95SqlSdAwIxI6JIUvu78u6vhhxCwrRaKGu/DU8X+Mx46aN0Dvcy2AA5g8pCxQTzLEArLH2pFzmreQGnvu59TX/MUI5aNejwZsDT5cGhjZPvdyOrk6XDk3qxWDjIUalYlSOvtT6BiMp5dSiEEoBaGjxqAEDjXnuGMSBKb14rBG+E264VNL8JJ9l6hH7bnXvZKx+c1KsKZ/wSAT6jqjwzf/4DhLQDwLQA+Zl//PIB/IyI/AgK1XwHgV9/yTSwG5IJY5ke4sQqKsV5ZEg8A27qhakUKg5Yt6I2Qg2Hq5faQ2sYVwr8pxvZ+KSVTtieZin/zkC7rhnXdgMrDNU0zsROA4j2HQxO59rSkP2h/sBIEtejQ4Y7d5ph+3dg+1QHQzchmksgWjr6JwL41Adi2SomHlJjVCGxmVUs3LkD0RTWsSu1wlxYe7Y2ONuxqzCZ4GFUr22ZABMcj0+ylFpPhHLIYYCZlxHScGu8eUAjScJFi3kiTaJTOAhZQ6Htd6WEmiQAmKKpxPWhQqxu+6rIWAGnpFoYYkF6y9SkuFSGO+BraurmxSabuxucXUAUQM+AAQ1/UiqodFJ6mGSfDkkhDKEgRkEocjJwTL4rV5sH5syfNYCgtkdAAakIB0n7OcRj/DKqtNrkfKfWDdXseio+346l8PYC/C+C3ReSj9r1/DODviMhXgTP/AwD/AABU9eMi8u8A/A6YOfrOt8r8ADDX2ptj8XZwR28KAcfDhUkhAuvG10wpAcF6JwPYTtcmYGSHyQoRicZzo0EEMU4QZbuFeZ7x7M0ngBWDiSrilFBztroi3opicX6apsawjVbez7aiyfgWnWjnh4QHhGQ8wFKO0bzrCsxptgphEsUAM0hRqSBmxkQq5QYksZufGmSbPEsUA4pybcLZ5uKUubEVJJ1VwCq1UzO6OS8MESrT2SR/aauITSkhbw56K+YDwcpqUpzZKowhwuK6bCCn3fZaKaoloWfhPCvloeroVahWHA6piW1XTYiBzzjGiTq22Z6RBEo6WgO4WmsjIapyDlUFKqwyVmXaV+uAQUEJCtumiokKd0EEB5Bj4mUSaU7YtoIKvi5NQJgONlf2FCLP1WRIzTvjgw9m4B1PIemQznNo3iQr6rvBBoJduAW1ZJuJz8g9c8D7gN8mODuOt5P9+e94sbn7T2/xMz8E4If+PBNpN/uQwUjThGmaLctgjbK061IIeBOtK3v5BmGhlSrpzX5r8GlSHNtv4+PhiJtn14AdzOrFeLWSPStdcc7DCgBGubZb3SjcMYQWbvkt7TgCgJ1nMKwRD3Ph7coKa791KINY1eP6CA0BWigMNU0TTs+ukUJiFkGC8Rks44LeDwbgPmbNiXsQ0l7beR0EZMOw/sSTpYG4/vkANMJZsYLGJg6u/B1cx0KBa2tSX/0+1V7P5enoEUPo68RDN3JhPFXtHojWSm/APdzQvSt/DQBr7VGNs+feJIFvXjy1rcXIgSImxt7MKUaUvLV9x/5B9CBCjDgY12ddV+QSUAvxntQ8PtIlYtx3gXAPbh82s5tiSE4UBFoco2genzn19GDMC9LhuI5Zpdsa94NRq3s0211vFmZR2pBZkdgao4tlZLZtRVDKB3o6sLnDQzFhMoA2bxseXV3hyZtvwmNi50ioKp7dXOPi4qJhFedkOsdSdilSGO4+4Be+Od0YjWMkKlVVFFXDBukiC3B2uEm3lirsGuDeBwQiEbVSkJmbsQwbdf/eTEkTXPRN7NIEGBibNBzB2mVQmwXa9WPcSJ5OJ7tpLfQx1x3w0KL28ogmq4jnDMjorbSG9x5KiskwqiJUAvXQbsDTNLUugdqIY91wjGvgEpjEcrpmcUixebJBewvbWoql9lk2khKV5Yq1TS3VL8H+Hr3gldrF27KQiYtqDoWgohj4KmfGsv9/xw2Hy0IGg6sBguEi9ufbLtH9+ZIX+gWvZtwPo2JjXFyn029WMRtjbDorITBluK6n5u4WP0gmkl2hLPhSp+cTtX/8+DGur68N5DRP5HDAaV2wrisev/Ye5Lw1D8U9Fuet9MPrKH9v0TB6WwA6O7UdVr+p+40N7On3zTCJ1yJ5Jonfn6eJGq4ISKmnGb0kntR+Dwn2coPuErP/TwdYO/7iwj5meBCMqEbK+phlG0M8//zPjXaxcg0NfjEMwX8fX+GFgOu2tXVr9T4iA95TmLFLEVLM4wgVsaop2A8KdTEipa5UH6cu/NWMrrG3Q3TdHPMWPElQKFsQAoWtRtwppoBtyztD6oRL/jEyoSkPbtZaBRoBM6xdUW6U/bALI5IESvA5I0Y3xO3EtH2kZ9IGY+Rzuwnle2JUPBXmD8MXVod/cwGjIKZSti4EM1WB2kOMBiLYCCFQiHpZ8Nprr+Hm5pqpXBvH4xHPTjcopeC1117Dsqzm6k4QIqVMdyMgGPXagTMeJJe5LG3e/r7doAzpWQFgN0yIwbRgYjNIDYsBrIMiMw0hACkJUow4na55CGoXrvbbkj/P/+ecsDMunJfLcnYjBgDqzelDbH1uqBhWUcHOjJNR+ZdloehRDCalyI4Du5DPmLrBPI8Y6Hly3aIRunzFSEfftoVZsIEd6gY9pYRSQYX8bdt5jCqKGNwYNYGItq+C3fRjNqiFWtJfB2XaX5WFo6Xm1hHRDUBMCWVdIZDdsxupBJ5JA4C8LazlCdbe10LcMeMDDORPwABnbVXiHs65sfQMXDtBtQxtgInH6WDnbxNfuRdGBegx8BgyOF9jnmdznfmanDOKHxh0HVtY/Kgmo0CDQur31dUVloV8EyibjT96ZF6Lsv3FuqyIkRqpY9c558q4xkoP1axPrnZW5IvKBkrZdgcEQMdsBvfcPaJ+CKS79bVLWEKt3UNwABrGaLU6EYSGdbhpPueeAPvbbOdmB2Hnv4YrRDgxLRq7VaJY2hTWj9izE2jv6W1G+0Xhv0uagJF7fb4mvg7O6OWcHOsQqAGTTmhsdVWpr6N3OeyG3TxfYDcfMW9wTM/6BVZK8bbdgBZs2wK17FOK065OjGvZny/3Dr/H962QSiCamj3BuEq1eXxuWMZ948/l/Ex41kzVcRrqrqBSe5ifA80433b+534YFXcnh2rWcbFdk4QFflt3tSPBs11Zuv1KMabrtmVcXrL69+mTp0TmSzZtVIpUP3r8mJs2JRwPxzaPpm0Ld4C6pit/jwGVACBxt1nHTTbeYv59v9GBAWMwzEirNVOTDqh6u4Za+o0k7BcAgfX4kWCgMbM9UO/RO9YB9TSjGH/CP1tkSXVTG6vVQGy7FR2cLrUiHWbDHVw3OLRwxp1FgbOJG9BhqVq0tfW1GdeDDb4ogxmt/SdV3CrD2HU1QJTeSgt3UmquzxhKi1gmxVO1EptwUrBnieHQNs8BPfVda0Y2PEZDasbHMaYx9GVa2g1kJZnPvMJYIhX04booLj7e+T3NiLgHOewbvqc1KBMLbNRT6Zw1sRdphYmiAdB3GabiIc6L6lRSYlUvZQRXVGNthkiVOIFZ93ZoPGZP3PxGcvvc5z5nDEZSslUrclVcXF5imiZsOVvFb3//krM1nNLWwgPOjK0EOQU0YD7/czLXaCTdII2uqNfwyLCBcs4t5g4xmpERxuawcCmoAXDU5Q0IDbBTE7WGeTW1movvLT89s2OfRQLxqiAC2NpVc9FVyYHZMWNTgoAZuWpp/DCEX4CxeA3f8vDi3OCKYPi6A6sEpkExpmAhQe3ZvskKCaeJHsUc58bYdWq/H07APaARVJVm8EIAUCjCncwoEfA3SMgzKlWRvY5LqZEczBC7JObOMNrvE0s2uGGdjS0M6ZeM/+z5HjkvQRHbfx4O8f0ME/KQWUfPtJ+v2xz3wqiI7N2+0RUHSHbLZQWqGslMTHrP9EJs1UIgQ8MNSa0Vh4sLvP76n7KiuBYcL9j+YVs3XFxe4eLi2ASVgr9/obTkmJad5hmidF/V49s6NDmLcbcJzl3YnEs7RCPIeXl52YwOgLap+BnMIwLgNSkhCGCcGm0GQuGL4Op0HWOpjaBGz8QB4tDUyIJ5Fe6x+MEKlioO2kMCtTkqYPR4QRL2O37eaEojqKU0tUPEZ9VbroyXwfg73APYkfVqxWqVxEECENH1UOyAjmEP10Dh3Rsd4+lrVJGsubxfZN4lEW4A+Vta61cBabIpsg+3z7PVRkVWs29lQwrJMmSBQt1gGO2XwwjqL8uy8+BgZDe/qKoWxBZG0ltat9VKOjqjnODt+SF7m4fxHRj3wqiM+LRvnMgrF9tazFsAi/usIVOQZHwMRTH2IMFFNhTbcsbj1x7jdHPdUrfz4QBVYM0Vx8srXFxdse2Gezq1YEVpN7mnVr1RWSkZJeeGIIdI5bmYUuNTjAfEXeFiFH8Rr5jmIXUleVK4PcaurVCyAihLT7EqlO0WVFukXC2fLQioajhTCixOa4phnScBDNkiuOwBXe0KM0DWJaCnawSlbsbfCfaoqgGVqRVtRuOQZAuJmGFSILCnMCq9k2LGWHQfJp4zeX2MIWMQoTtvnk6AfZYA9vrB89R//n7De5RtST0T5riNzwsC9uJRsAVv6FIOIWfUbSVlv/gcGIJvA5OY9AeGWbkUBAWmNAEptT7HABruB8AIn2JV1s8nG4jLeJLAw7qEGYp1VdRsdW0SW9Gs9z2C9PN1G+OeGBXsNhU9BooJazM47roOh8CJaqG3jIymrfra49ewLtSNdZX5ZG0lpnlGstaX7tKqHYQaOmruBYHTRHe7ltqV+ULPKvgDGw/FiMm4kaCbzDS0N3j3MvsQ1KqGXYtUUIeeQ2x2FpFrbgQ82Dp5rYqAcXQpFu7EiACmNJ36LsGJg0ZqQ+9kIMncebtRdbglm0Shke3EQHEPp3pa2ng75hnGNPE15h2VTM1Zd/vHMGX08MYMDd9XG4YjVhuEBtB7+xRpQud7Zq5jWhTiDto5x40bZABztVCSAJZC7aKqeWONzkbGbCgZBWKXSyfoObBaq4mqSw9VnPO0Wt/tXDKS8HsxRGCmAXAlfQwho0j0FW5nxpnEFPjebP9Yq5aWGfKz824zKr5B0TdBLqyDCZFszgDT4BBqS7ABFG8EoeQWpnlCKSuOxyNK2fD06RNsecN8OBI32Sj5iMB2paxKZusJ9ZJz65OiRlUPwhvwdHNqdPSG6VgK3Oc9uuojmc+H81oc8/GMlQv9aGC2wMMSL6QUMPypuaDkLvrdXXgHlA1khe6MgXcA8BJ68ZYNQ9ztgtil8YIGEpvxLBjPzwMjtAPOas/RwXIRsplHn0OVLWOPKcL5Rf59oOMuI+bk4UR/Dd9NmkyneUMyiJ377GTUHTHjObyX77Xadp+0z9DmYocyWX3QtmXkbTMMyavog10MAxgfBhJnjC1ruatFA6xAdGqGz6vJPYxp5Lh2iXCM2B/D5YhilAAt/Vk/FwbdwrgXRoWbsIOUthx0jwFEMA62U9SKx5xg5E2u1HCAKSU8efImct4wTxOOxwNOpxNcS9T/v7nILZ0trYm7KhDTjDlF5G1BKZsdQmGWJca+wWVPIx9vSh8jUOmALr/vN1wv9gOAzVo+uCdcsskpAkiWfdDKDI8bXuJ4AoSuON+yTyJ2Zuzg7QzK/jB6StQ3Nd1/sRBP2iFppMBWO4MGbMuZh8EPAjNc2Z63A5hjZqrjTo4BAQOA665/8DAM7W9+zr1Cmv9s88gcCEdP1xq1g2CnhX5N6Q7MLHmo620/tpxxMKFr3TYkEwH3tfG91ZrcG6Drz9/3C+dTEGPnWk3TZMAzyXe9yLBzdsY9xc8OuOqep9udRXubXgpwT4yKQGwxyOeIIpgmS8/65rYHFg2ADBZ6VK0tNtVScLy4wrNnz7CsC0IMOFg/YZaSCw6HCSX3AxEltFoJVUXNG/e/4QOAYjUt05BYss6Hyvutl6Kj8VhGzoP/PYKPnuVSO2jORRnDgFr4J4auEC8hYD4c2LdI0XRbgwI9LvOeMepWBhJZI+QNr9xoduDSbsHQPS7OezCWA/aylWwh0mgc+xr672W1bqbAljJbFZP/zu519LkAXlhaKyDS27LEaBolUIhECHpGRc0uNL4Supcygv5+QEcW624fCrNMlpM32j8ZyhXE5Byf88/Kj1WbFwLQKK/bBs96bcYSjjFi27Yd9cAZsmPo7IbFWdu11uZlj7jduO6cDzoWZ5eUt829TcPyfGHKHQxnFzrDkIfObr/qPIiI2SjybQNXFxtiSu5wOOLmdNO8kMvLy5ZqdJc+pYTV+smIxeIAH06xTnO09J1z4KlK74KYswG2GG9VaZmHMSwaxzkpDsBzuMu42btIEQ/4PM84HI5M6SbjroQAk4prwuBUF5vM8BrLUijt6NjUWHovAoxCUr4enDPDTrXnpOa9QPYGsz3LMesCHsaqnpHwymTZrZuHBiMfQ5UZOBd/9vYhpbIdiAP6tEsWonUnrBl7hmr72rJdeGO/x5+5l+h4OOkdFgkpBUzzYYefu8hIHwAAFfdJREFU+IH1S8rXL0Xq+6IqtLA4UJRh/Ogpna/386EgPUT3dMbs0EhZGJ9BNNhA63hBvOswFSM8BRkOpNoNFeC1IswAbGaReQimw0wdkshq2OV0QikFl1cXJtakgC06m3ufOqch9XRgNSDWEbycMy6vnK/BcCTJtDMyDr7ywe6lGoF9yHPukp9vDH/ko0EZQcZgCvOqim0zoxiI+aBWiBCniCF4FTxqpUHp5CxpFHx7sxb6KPymY3vQF23CvRcAQIeiP9mn0hvxLXTqOtDddxG0wzWGBgTfGfoQb4CFA5vd4l5HRGqBqvVpNu/HD3mtLieQ0EsqekkE0KuBzWHsn1n7c1ABoj3fbMY0pQnruvTQ0rKHI1XAvSOX+BQRagOFACh2pDnq+DoOJWg2W6inEkQQpgnIpe3Pc37PiBHxeQASpRE0b3PcD6MifouPt97Q/tRje7UMjAAlA9Nh5o1WC64uHuF0fQ0IcHF5hFaKI/mm9o25LCc2twq9vJ2q+DRi1bIcB2sZ8fT6GlvOuLi4bJiF9/d1SUVvbeG3ht9aAJ4zMjuXW4aNXTv+0WJnCFmiipYxYcp5aoc5bxvlJAUQSUOmhfiJs2A5GYHX1PDf6U343ELbpL0I0ufZ60pAELCaIY0BRbXrrlg2RaWzWIGe2XHvxNdq7KLo6+WKdsBZe5HCxugSqIBHoFcpMj4YjvE9/fDuDLjs1dHEwN7m+QBAZVNzFbANh3i72MpMY3BhKALsvmBjuYmItASBE+RaFkt7uDN6d88dDQsjpWU/95Xw/TP1n/EskQRBhOz2422M+2FUbLTbPASLnwfrO8bAVVttzLqseO29r2HdNgOrSDXPmQ2gUDNrQYQq7xQ4sveyjejud3URIQmYpgNOy4LFY9kYgVEbQ7VtdiqFdT3VscTgPNvgXwNoBs35Eu7pkPMRIcFJVZGeWO2xPkQhxofwz9d1THj1OgelSiZoZypsztkYKfquD+xehAOltCV7IBfaMQsdPg8AI5Zpf+HZ8wUGzRm7qZt+LTpYadxguFYLvRIykA+Hg/0eRRBFlQEsHsIqN2LAPjXdPsaAjfQsi+05AcNLyyLQAFPkCzi19awmSSHD8x9BewBNlLwZG9Awu6cWvIYLINcoWFpYe19wVUUK1P0Z64Gal127SBOxoNDSyuc4zKse98eoiFoK2ZlHLZ3BLnniN5HVQEwJuRQcLo6ACtZlMWEm4Ob6GqJM+8LcZ8sx2YFNPICFRLe8LiiltgZjXs16fXODFCKmNEONzes3Q94yqMDfhaZHfMA9l/OYdmdQ0D0EWEjXHr4ZgClNJsLU64dSimwBGwSQ1Dw8VW1yD0BvKK4aYE4MgIoQ+Dry2LSl37w5OMFvUtmrkdw8q+LatJ7qtg/1nNHsIGafd/+sfN1IFhtfx9CuV1SnZA3lA59ZztkEkzKcBlhy6fgRLJQTryEja7YBu+ZhOAFOTZICqkbPp4sSvLWo8mekgqlsrlpL9auyDovPdkhVDziOeyyqysIKq07f1/z0wstaizNMeKFi0Aw2nMl7N0m0pmbujdhzap7lS7ygVzXuiVGh+C/DHZhCHl3pAAKP1XrPhCCI04Rst8nhcMDTJ09bZujm+hm7xVlCQ2LClBJOy2IYjbWyLAWqGbVm1LrCqzlVWdkMO0ge6ggl5RBiZFtTBMzTbFklBVCfc33HTTU+2HYARzxF+vc9tPE4v2J/2+Rc2ecZkc2uhgNdCgHBOU2NUh/jZLdtRYULGlGTRSBANYMmHaCEmN6JrX83imzRGpxEZ7H/iwBbx0l8kNhoxmQAsv1Wb15ajKzmhT8vMdFv+/lSqIQHYMsZs93+67IA4J5QVVPz5+ur0BQ0T0FZ1qNmU0vNUJiHKFTUF+WxBqj6K55ab9olBl9rAaunPau3L0z0NXCPJcAMXmXTuJILpMlkMlPnhj6AHhMxZG1kSabvLQNm4Lya0hyKeVmDLXnXAbUOao3p1gZ2DSQffy0A5G3D5dUVbq6vW6y9LqfmqYuwrP14SFb4VlpFK+NZGhS1amOtFUUq4swWnM56VGjbjDFKo2PP06G1DfWQYnR7Rz7B+WcFuiyjf+/ck9llKrDfFG1dtAtAj16RSxCqFeHFQZy61J61al5TGAUIOfzWawZyAAHdC9PaBZ3HubffYYdAhrXItVpJRH3uc/cQsr8+SD+olEDoYPCYEfE5eYdDDz133o9UqHo2zGZoKK0qK4oRgZS6wWv4lHkQLT2EPTjKhMD4jPseGPGi1Kj6LFqU5rX5GoZmsO0bw3qig+u2h4K/n+FeQXsSoA6vffcZFQiSBP6xVHJzuQFD2kvLsDBnT+bp9elkTdG9qyA3+7qtTd92WRaGQKooZUNQViuXmrFZKYCEgClFXF1eUsB5Zec9AdmKIbLHTC4Z03TAlGaM8n7nEonPgbLYG4qe5o47V9lf55vxHAPg9xNKUeStWANvNW+BLTTcXU7Rq5BNq7cqNDMsCXYj9lh8j1sxzOqi2BIor5gsVb0jvWlXPBuB1Ri7QLeb16IFUmEFoc+X9zufJIjrpihSCi3DU6vs3rMVOg7/v5o0wvgsWqFiE5sG3NNwRf5SXBKz9tBIfF2MGAc2eJfBsMBSz23/6V5c6tx41kCqvo/g66AO1j9/ifh0q+7Jb80bNmBWqjRsTLG/vG5r3AujAuCl3A7nrzDd3KtYj8ejNSinq5u3FTC+C5TMyp6K5A2Xc4FAkWtmrUTJtskYIpFGH7CtTF96Vz5YGnNdN8QpWZtTpps9WzQagPHv8aY6NxIv82Q6yCdQ3eu+8m+LmdWMCIzzELQ19O7p0m7gHAfy9hOoBSK9kfno8eyehzhxLzWZBjGXvKW87U8DJMU1QdBe65XFVVjwFlNvV+ufYTZmah55QMZFGg3wOVnMjdkornW+p2plMaECZyGM4ztDUSdz9e01DprCvFbFPqVbVYHihLzBgLzAY/U5Nr6UCIKRbPiM95mqnsAwNrl2AHYkcfY9ZM9FxFHgPdD+ise9MSrAmHsHaCC2Zv3HIr2LiwveIqBoz+nmxLSwtc4MQvV0EQFyRjQ+CUpB0YpSaFAMJ28ZnBAi5mkmC1Q6ZV7Aylu/Cacp0cNRGDj4/I0wGofRTfbPOX7ec4Cz31JsoREC2joQt9lQSsVkjdWATqKrtVrJgkJA9bcQ+flr8U5+diOa2+zyie3mCyx12KctA+b50LgVnY5+Bj77H+NXqBsVnHULHEKYDkAOKm2DhwHwAF5eXjYj5qCne1SjcXJ+yPlzGOcHK3VoBrK6IWbIo/CiQGk5LD/wwT0V8wJh4WHF3oj73x6e+XyA5w0LU9Q0tmMvpd1Fqz0EGvcZ6RaDx4m9N2cw9K2Ne8GoRctYdivdc+t8sONm9halBL5Wk5ckfToFZhimeW4L7YJLubAYrOYNGFpaHA6XmOcjjsfLtqHbrSFioVVGTJH9bAw/8RRuP3h7AzFuqtGonItGvyhbMGZw3COAZQBIqKK+rle+eqhyOBxae5KU2A0gReO6xIT5cDB2cLRGZN5SNUADQb9pZtGl2k1HjkZqhpoMz4SxLGHkhjQ8JHYPhqHEyCoOzZvwFLEbA4Yupkkcu6e3rgsAco7YVL2/v3OF/CD61+459aJINM7T+Jz81LVwxVLajX4g/vnsM8he/NuzWh6Gt6097IHdfjftWd9v7d91mBOw2xuOr/nXY2axgrwar7nqn+s2zQnHvTAqAgzEq76QlliACNH5BsiuDHVYtWv0aGgDqVKK1FTZ+IC3bWP6sVKSECAgJiKYEnsLxThjmg/sSKh9M7AxFG+SaKltBxnPPY8XfT123wOeB5zH15/H3n4L+aHwDauqOB6P7RbzEMCzNOchib/HNM1WP9LbPbjhDIZx+I06ut+NQWwptSbbKNY+w4DFZVkG7kVoSmqAkcJqQYiRhi7tOwg2lT8QtPSw1oW4onlbvn5uRMa189DMDcvzeMZ+jd2SvPAZtIuOO9SxE76/mOiUrZNhci/iwozPsX/f165XqzdZUuhzl87L6PzjZdb2o4VJ/XNIy87d1rgXRsVH4zCYQUHP6KGU3Daeg1x0pe0B0C8EwA23mpYtb6oMoFqrBLqHQXgIp+kAYhd8623LTQV99B78Zleg0f9HL8Vf76PW2gR3zmszem+YfVuPHfsWAdB+y+XMlPq6UtphPhwBS/cSwE6W3oZ5Kb2WyLGGyeqXQnAvxd43WkdIfT6W53MJg9exF6NyMWxiTmsPwWy9nIwIkEsCC02TYVN+gNxb8e9N04R5PjQOjohX4XbNkuPx2MIeNySHw6HNvclL2ODn6wdUzTP2Oig3HvwZxhpiz0IQydxFMI5Jx3j2F+A+1B1Hr17e4zGjYXHDtQsl/XfZRznHaNwI+mcan19HeG5v3BNMZe+ltJhbvekVK4APh5mq5gampehxtJq3Q4MCEazriak7F/XZPZxAYez5SAMifBhOglJoyzBJ8HJ+8zy2Dc58UsLre1ap9M6GTOcGq7Xp8b9vID8gQN8oPf4ndZ98DW7YnBmCHY7Hxt1Q9bopP8ysLSFhznrkBO/Z6x0bsXsv7wjgxvPc2xLzOiQ4l8dDM2tivy6sLB5c/BaaeHuJjf1zanVVNG73nDOWZcHFxRExBkSTE8glWwuOauzl3BjCW+4XzDRNWNalEQ7dKI0UeK6x39j7nedlCsyi1WEfAtRqMmMDGbJV2gwmpTWVrxOaHwhJeiLxLFPjRju2PRMda1GmgEsp9vNjyt00X8xySbcg9nzM2w8mRO5b0U5Gu6BvadwTo9JBwnFDMyVZodVu9wj2zy0FIUxwarmINgZmSgl5yajWaCwEqp81AhmAigBJB4Q0g7w1xTQFlLpCYsWaFzQpPqB5TmLhUEzW6Q7MxMgQy9Ig8semROZugMkzikCElbalaeCqHRb+TIvBK/vlllqQIns2iwgOxuBF6ADi6P5SnoDZieo3behcD9XSDEMQQR02G72dzndxMlecCGKLym7D8lBtUJgIs/ZOAxXAIc1mSBQlWa/syufNfkGkvW/bStZGBIoq0jSjlK29LmpEv91Za7SuK+VDY0KJBbnwT0qU96zmzYoI2cNxzEQNWIRhccRFvGCxf36uk0JRrP+0GxZ6h16E6rKW9ouhYU8zYLjddLTtohEahFBb54QRP0P7L/cgSi/GjJapUnD/Bj8LY4bIw7i3eQrfqXFPwp990VPPmoCAKmCZg2JVyrU9AJG9jkatimVdG9VcoU35zDcVXWt23lO71X2D+a03z9bu1B6YA2TtxhrwdGMJ2K1BlTDxkKpSMKnk0lz16gJHpTeHAno4t20r8raxybfdrnT/e1V1j7HRPAznijC0ItvV28QCzGp4pqDjPBbiBHbz62vqtThMt4/4RMdhBNk/i+m7tpveDwAoEM10fceZ0ADVqRU9Os4jRgdo3qCieTae/SAIy4yWeySt6jmEzqWxEgcH1s8B9XPMwvGmsZVpP+T9sHvY6JeJg7cEeXuVuaeM+TP7TKE0wxKb+NQoeMWLrM9hDJeh7R/2GM6ZCWlh0LsPU9nHnx3UrKSkC7MP21pMlZ6O3Rh7epyd82aAIG8UBbMOCB1YnOauy8KYn++7bZnZB7uZz/GScYP4JvZQzV83T1PjDHiqNhdXMSPoOoK1+wNP/ozruwJdzYtfu0Hp6zRiB3twsuMfo0vfMldDnRU9vN4I3EFs/39HFrrh7pmvcXMHwz2aAR6wncPh0JqljxwTFxX3HTDPs/1bMu80GHmv7g+jKrZ1bbQD70/kqeQ4GMHW2wfjhdUP2flBHFndw6bcea7u6TVvp3Zj4/tjbJE6/m5/FvZrEUyZ3+cbYyTTtnYjhhcYl3H0boUvHu9eoHZ3APwQa0u7res2vPp5oDQ4H8WeRd+E/fem1ONcB0sBIOeCdV3a5h/BsBHnad+3y6KnNQ1wQy8qbBR6MAQZe7s4VjGGfZ7+ZOjS4/tivBlfE+/s58ZjLG/oTN19uYPP3b+XUmo3a8dRZPg83WB5eObz8ffbrDK8eoN0Bdz49OpraWud0tS8pGxeWghkLDtSQCMwNESPVPiLg5H30EFVsZpXZw+KlIHxWYmznPdAuj/X7oX0Q7df374f+87rf3dgthv78d+o3NblJbu3V3f/3xrSQ1vXCDXDImf2YrzourHQF7zHiwHjVz3kLt70uUmI/AmAZwA+e9dzGcYX4GE+n2/ctzk9zOetx19V1S981W9yL4wKAIjIr6nq19z1PHw8zOfzj/s2p4f53I9xr8Kfh/EwHsb//+PBqDyMh/Ew3tFxn4zKj931BM7Gw3w+/7hvc3qYzz0Y9wZTeRgP42H8xRj3yVN5GA/jYfwFGA9G5WE8jIfxjo47Nyoi8rdE5JMi8ikR+d47msMfiMhvi8hHReTX7HvvF5FfEJHfs7/f94rn8JMi8sci8rHhey+cg3D8C1uz3xKRD93SfH5ARD5t6/RREfnw8G/fZ/P5pIh80yuYzwdF5JdE5HdE5OMi8t32/btco5fN6c7W6V6M8xLr2/wDipT/LwBfDmAG8JsAvvIO5vEHAL7g7Hv/FMD32tffC+CfvOI5fCOADwH42OebA4APA/jPIA306wD8yi3N5wcA/KMXvPYr7dkdAHyZPdP4Ds/nAwA+ZF8/BvC79r53uUYvm9OdrdN9+HPXnsrXAviUqv5vVV0B/AyAj9zxnHx8BMBP2dc/BeBvv8o3U9X/BuDP3uYcPgLgXynHLwN4r4h84Bbm87LxEQA/o6qLqv4+gE+Bz/adnM9nVPU37OsnAD4B4Itxt2v0sjm9bLzydboP466NyhcD+D/D//8h3vqhvKqhAP6LiPy6iHyHfe+LVPUz9vX/BfBFdzCvl83hLtftuyyc+MkhJLzV+YjIlwL4agC/gnuyRmdzAu7BOt3VuGujcl/GN6jqhwB8M4DvFJFvHP9R6bveae79PswBwI8C+GsAvgrAZwD88G1PQEQeAfhZAN+jqm+O/3ZXa/SCOd35Ot3luGuj8mkAHxz+/0vse7c6VPXT9vcfA/j3oEv6R+4u299/fNvzeos53Mm6qeofqWpRltj+OLrrfivzEZEJPLw/rao/Z9++0zV60Zzuep3uety1UfmfAL5CRL5MRGYA3wrg529zAiJyJSKP/WsAfxPAx2we324v+3YA/+E252XjZXP4eQDfZhmOrwPwxhACvLJxhkl8C7hOPp9vFZGDiHwZgK8A8Kvv8HsLgJ8A8AlV/ZHhn+5sjV42p7tcp3sx7hopBlH63wWR8O+/g/f/chCR/00AH/c5APhLAP4rgN8D8IsA3v+K5/FvQVd5A2Ptv/+yOYAZjX9pa/bbAL7mlubzr+39fgs8IB8YXv/9Np9PAvjmVzCfbwBDm98C8FH78+E7XqOXzenO1uk+/Hmg6T+Mh/Ew3tFx1+HPw3gYD+Mv2HgwKg/jYTyMd3Q8GJWH8TAexjs6HozKw3gYD+MdHQ9G5WE8jIfxjo4Ho/IwHsbDeEfHg1F5GA/jYbyj4/8BnQCjc0iHc1oAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "import tensorflow as tf\n", - "data = get_image_data()\n", - "req = data[0:1]\n", - "r = sc.predict(data=req, payload_type='tftensor')\n", - "\n", - "preds = tf.make_ndarray(r.response.data.tftensor)\n", - "\n", - "label = decode_predictions(preds, top=1)\n", - "plt.title(label[0])\n", - "plt.imshow(data[0])" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAC7CAYAAACend6FAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOy9WZBt2XGe9+Vaa+9zTg136NsjGvNAEARAcBAgQAQHSUGKIiGRCouy5LBFO+RghMODnhyWn/zqFz84HJbCkDWRQUpk0CJFShBJCAAFASCIBgmAABpoAI3uRg+3p9t9p6o6Z++1Vvoh19p7n7p159t9qxuVHber6ox7+HfuXH/+mSmqypEd2ZEd2ZG9uszd7g04siM7siM7sltvR879yI7syI7sVWhHzv3IjuzIjuxVaEfO/ciO7MiO7FVoR879yI7syI7sVWhHzv3IjuzIjuxVaC+ZcxeRnxaRh0TkWyLyD16q7zmyI3s57QjXR/ZKMXkpdO4i4oFvAD8JPAE8APwdVX3wln/ZkR3Zy2RHuD6yV5K9VJH7+4Bvqeq3VbUD/hXwcy/Rdx3Zkb1cdoTrI3vF2Evl3O8HHp/8/UR57MiO7JVsR7g+sleMhdv1xSLyS8AvAbRN88N33XVy8uwVqCK92ivGZyrjVKknVQUFRUEV78CLoFnJqsN7VZUpXSUiiMjaZ02eRBBEQJwDBOSSfUXKg+IERMiqaFY0Z9Q+przW2fdTtlUVKe8TJ4g4+6TyBkHqVqOqpJTpuo7VqqPvE04E7x3OCaEJiMiwD1L2WMG2Q8F7h4gQYyLGhPe2TyklAPqYyMm22TsHKCKOnPNw9Os+Tfdfdd+xU/Ydp8udUbn0ufJe1XoYJh+kAqLUh0WEnDM5676z8tLZGrbb5ofvvnPEth7wW3nX+mN6xasAO9/jT/tVC+gVJwXbBcvDNVCwX0+FiOCcoAia83AktTw34NuN18DwvH3A8Hh9vaoWfOfpMYGK1QHfOtkGN7z/uefPslp1HHVGuTZTPRjbL5VzfxJ43eTv15bHphv0YeDDAK+9/x79n/67/3z6XPmZqU7LoLTuYJV1h7HmxIGUUrmwMyllUlRSiuQckRTZmsFW44h7kb7vUU1kzSQyOSU06QDetm0JTYuII8Vozk7BOXMeznt8CEjwSPD40ODw+OBBBO89TszB0gTU2+MXz1+g7zpi6lFVvARUHDH25JRIMdI0no3NDeaLOSEExAXEOZzzOOfImsmq5JzpVj0vvniO7zz+JOfOnif2kcXGHO+FO++8g9lsRowJxZx3zoqI0HU9qo7FRotzjosXdjl/4SIbGy05w95eR9dHnnryWXZ2OnJO3HHqDrrVClVhuero+56cM6uuQxSUzObGFhnh4sU9VquOnDI5QS4XukhG9kFz/40VVUQyIKR6cwZQjy9OwRyJsph7trdmHNtuOXZsxsai5XN/+iS3yK6K67L9A7Zfd/89+vf/+79zyb7dGmyna8N2dw3YbifYjgXb/vqw3TQN2gSYYLvrVteB7YaY4f/+x7/Ntx6+Zefsu9ZeKuf+APA2EXkTBv6/DfwX1/LGyyd47SJYj/6uEL9PLoYhctE8iW4g5UxMSs4JUItwEbwLFlWn0WmuVitWXU/TtAAEH+i7FV3XE7xHEBIRL4I4x2q5xBOYz+f4JpBztqg8JlQzbjajmc3Y2tpkb9exs5uI3Qolk4E+9uQUQZV5s4EXVyIcbysEEZzYiqH6OikX2LFj27zm3rs4tjVn1Xc0TQPA1vYWwQdSiigJVXMSznlin0GEtg0oSvCepvHMWk9SZXNzwe7ukhzv4Ny5XbIm7r3nFLu7u+zuLZkvgq0Yuo477thCUyIEu3AvXtzDS+YCmW4V6cspEPKB0ans9/YiOPEoDlejdjJOAk1wOBfxDu666wSvv2+bkye2OXFsk/mswTvhyw8+dyXIXY/dMK7hNmM7FGxLsPekTM6XwXYI9Kt92NarYzvHOGC7nc3Y3NrEXTO2Hecv7PL//vLv8fiTz17rIT2yK9hL4txVNYrI/wD8PuCBf6qqX725T738qvqgi2Yt6skZNBd6ZYiRsMUotiR0Dh8cSTOpXBAoOFW89yCQM2TNxNiTciT4gMtKjImu28N5zwxwZJzzqCZWqyULt8AFT86RPim+bdE+QtMQfGA2n9H3HbHr0ZRIMZFiT9ZEEzxNE/DBD0vX6s9t++1vL0LUjPOOjY0F6Am2tjeIOQ2US/Ct7bdryNmOT84Zh7MLXXU4RvPZjFkb8D4Qo1Eyi8WcWdsSwgs0TeDkHcdZ7Hq2V3OWyxU5ZRDY2twA7EYRguP5My/SBGVj3vLcmfPoMpL7hFJpAFmni8rv1ck75wjekaKWk5dwTtmaO7a25pw8ucF9997B/ffdxbHFjFnb4sUjajSW980No24fpl4CXMONYbs4+GvBdrsP27lgW9Si7/4AbIcDsC1XwnaiT/0athsf0MthOyeaZh3bD/zpQzzy2OmbP5xHBryEnLuqfgT4yK34rMrZjr9fGxk3RDYoqpmcC7eHx4nSeKEJjpTLa0RBLMecUm9L56zFuQT8zCKb+bxhtVyhokgItizNPSnZhRByAITFYgvnIMYVjgbnHSn25UYS6JdLwmxG0zS0bUu/WhFTJKeI5oSQCaEtnLltl/GjZd8oC/rCK3vnEW+rjtnMoqqkGc0Zh7foSiOF6QAUzSAlr971tqoITcA5i96dNMTY47xj1s9ofFOcf8t8MWdzo6HrVnTLzo5T8CwWc5pmhjjIuSeEzLFjC154fpfz5y+yt1KaNpBiZ25IL81p1DxHpQ28V9CIF6VtHCdObPLG153inrvv5I6Tx9neXBC8h2zHIuZs/DsZzbeOvL2VuIabwTY3h+1csO0dIQR8mGB7VbDd7MP26lqw7fFB6S7B9nLEdkqITLHtAfjSV759qw7rkXEbE6oH2X5ecYxoahQ3gl+H9zA87pxDyWsJQycOnMchZLGIRp1DNIBL4ALitdAUFgF55/DOoymjOZljVYzrFsE1nma+SSpJx67raFqPi7EkSSHFnqXusNjcQMKMbrWknc3AC33s8dkuptRHXAg0viG4QHIRQXAIOAjBlUStJclsv2oadXSIzntz/GLHwAWPTxbdO8AXp68EnLNIzZbz5ZiqElMCHE3wOO9w4tBsx8MFj/MBEVcuWLugNVsC11YHStsGmqYZbkYp93jvWK16utVzNLPAImVCaNjbg9iLbYtky2PUaL2sSGazlvl8jmqHJzFvG17/2rt561tey52nNpg1DV4cGjNxuSRli2rtxm4rA9V0i5F6/fbSYNuD0xvDdrgCthcTbK86mtkE26lgO++w2DoI292A7Vyw3fqG1RTbcnlsH9mts0Pi3PUKEcuYbLr0XevL+SGaMXVEyUoVvtYL4lJxZiAq4AQVj288wSkZS1BpzoUdFLIDzYq44vCcJ3tPCIHZwpH6nlmcEXPC9T2p7wk+0C2XxNSzXO7RAr6xCDfMGnKMxL7HNzP6VY9P2bwtAA7vA2iy5JX3VOSbA7XtEnGIM9WN8w7nPar20oTinJCSQs74psF7h7pJVFyPrpgSpu97lFQSxx4RIYQZKdrnI44MzESYzdrBeeecCT6Uz4k0TSj0UXkuBLxraELH1vYeofGcPLHJYmPO+XOe8+d6SwhmRVVAHZBxwGzecOzYguAdXlo2Fpu89S2v47X33cnW5hw0k2Kij3XJH8cVQLYbVsr5Cth66W1/YnTdbhTbmSpzqthGkt3IsgD+1mF7vg/bboLtvStjuyvYtmS3cCVs55y59+6TPPLY07fw6H932yFx7qNNZVuXJNbqa8rPkV4Yl7STnBI5m9ywKmbiQHeAd4IPGXWWi0JMBtg2LYqSYxrkfzFG+hjpVktyTkSBzc0tmhDIKTFfzGlUyCmRU8KLp53PiV1PHztiSvgQ0JSIK0VwpK7H+YaUK8dfqYiyLcHjw7pETEePPEjTVNXomPJ83fd6DEOhjabHcvgeigyuRIZtO6Ntm0Ga5pzDO9DckMUcZY0Yq/TRuapYAefs++rn1xuAPeYIXjh5bJPt7W1mi5bNjRYn59jdXdH1jkY9fW8rkrb15thD5s5Txzh1fIPXvfY1nDi2SeMVciT2GY0JckZTttVOkbZSHWG6vc59arcG2zVXMmK7j5Hf+t0/5OLOHgB33nGMD/2lH2CxNb8GbNvNccA2sLl1FWwvCrb7y2G7K9hOQ/6qQPgy2LbHj21vvCTH/bvVDpVzvyrwr3SR7pONFW1zuQDKv2Q6cDTT54xvlSwNmhMkUBGcT1RtOYA4z2y+YCM4iq9i1efJ1wVWXa6iX8QHEId3C1xokM645Zwj3pnksA0tfd/jfEcCXHBFY6x4J0RRQtPgy7K1OsmqExYZk43VmVITrYVLRTHO3I2J2JTSmKx0U++hltxq/eS7zDmrKC44sgqxiZjIIYFoeY3gxCGqqJqeXqTq36V8nhBCZjFvecPrX8NiY0E7c2xtzsgpc3En0HeRlBwXLyzxjWNrs2V7a85r7jvFPffcwV3Hj9EEh6YEKZH6nhy17Kviyk09lei01hCQ9TKx8ctrLyW2v/Gtx3nksdOGbeDpZ17gz3//Gzgxv+sasN2yGTYLHa8TbFu+4saxvboObJt3P3ny2E0c4SPbb4fKuVc76EJYL0SqSmcZLu7yJFU3rCWMtcKeAuoSTaY4cswxgvbGR+JkoDdyTogTYjRwh2CKmaYJhGajMMJCW6RgKrZOjjGy7DvatgXnaGczOqnbLHhflCqa6LoVvjUlR8oJ1YRz0LZGo0ihW0QYqBdxth3iGKLrcrCK5rlILYXBsSNStOGlcEq1yCntENYCrP0Jvel5sMi+ZS/uWuJMihLFmc45p4wrYaJzbjje3nvAXnPixDbz2QLnHM3cMWtngLC7t0eKmXNnd9jemOGDcPLkNvfde4rX3HunSRqzRZyxL/uYCsfMlL1WREukrgk0l4Tj4Yjc4Waxbcd3iu0Hv/4In/zMFwbHXi0lU75cM7aBpmkI7cagYGqb1m4iYpr8lxLbjz3+HL/1bz9108f3jpPHeOHF8zf9Oa8GOzTOfX/hyhUvyLpm1emlPX7OwE9Ofq9OT4dlojPuDzUnkI2HzymV5WQGJ/jQGO2BkPrEzt4uWXdwzrPY2MS5QB8jrvFFJmiHNKUImpjN5gRt0Wz8sDhH7Dqc9+S+x88aQPHeEot2oZn80gWPb5rxQnDeIm4oDv6AKLDst/d+cLKVotAJRSH1OUb6xDmrNJ3SB0BxkKmoMTCud6BtjKONdJPIXQpn7+sm4b1nc2uTJkTjeBvwrqWZzelWS/q+YzF/kZyEeeu5886TnDp5gsaBpkSOkdTHUoBTOdyyMSKoA1FbQYjo5PyvO73bYbcO26xhu+8TD3zhQc6eu3jgd14R27IP2zGxs9wh54s471ksNnH+atheEIoa64ax7RxPnj7Dv//oA6xW/Q0f45MntvmFv/ETnDt3kV//15+44c95Ndmhce777aDI5mq237HXAiRV+xdrBZ+CL0kngaL6qEqFsmzVTOwS3WqJdw0hNDRNy9bGJn1c0vU9uzvnzMGLI/WRjlwcqjnI4J1dCCJkBBcaUm/g1/pdMkbGWhKAIQRwQtO2uBBMrtYEk5l5X/To5ki16JMdpjWucZ/zpjexbal7NkZ/OaXiO9za8c05D85+yiCYw1bjVzUPdIwvKh3vi+qiyO1qG4NyZvDem1RxVtRLAjKf0czm5Dhn1S2Zt8bNb85nLNoWB1b0FTM5RVLuy7lNTJcdVQ6aUlzjqeFy6crba7cK2+cv7HD27KWOHcZU7YHYzpmYCrb9AdjuCrY3r4bt/qaxLc7xq7/xcZ48feamjum73/lm3v3ON/GVrz3KG15/L0+dfp6+jzf1ma90O7TO/VrMdL5lqQqWUETH6HP4jzEZqQ4FEpmIkiUQfEZ8SeyoIiVZKEiJdCJ9l4r2PUJomW8ucFVf7oNRBDXiVei7HmYtVl7viDkyn83J2RFjxDtfHKQflC/OC+ryEEH70OKL+sS7hlqpKViFrLGiETDu1eRuRtl4L6AQUzIKqYTcg5wulQi7aNzFCSq2slFCkU7a6qVepCoZ3wi4YMcJsZuIOISAc+Z0R67dInY7LhERTwh2ruwmolZARYMozLdb2rYBjZATse8hWdLU2j3UPEGllOzvXLCQB67YDoKI6UJeiXYt2D5/8WKRr15qX/nmU7z1dXcdjG2dYDtF+mSFRQO2t241tt0E2xaEuNDyzHNn+fgnv8iZF26eRvnaQ49x/sJFvu/tr+Mdb38djzx6mtXKnPvHP/kFvvHNx/mRD7yLP/nCN1guu5v+vleCHRrnftnGXFd8E1TSVfP6e8cIJw/RrIhDCkcoQ+JvfH0ITXEeCe+VnIyvdGKNtFarFTlm5luBVVwCEHPi2PY2IfjiYJWmaSzCLhHrarUyxUrhPUkJ34wqlqGtgDgkZ3wTLCnlHE0ItoT2vjgs22knjkwaj1lJlPngSzm4Xbz7I3EtuQY/JLLKgcwW1+VUNMeT1cHAzxc6RkTs2GCRYWFGiiOnnJRSWRrCkMgdtO8pQelxYhWyYpW4oUVTJqZsVEzKaIyFhhmjV5HaLI0hcqduS5FUZkDyerOr22W3DtsjN59z5uFHnrxsdPriuZ3h90uwHXTAyCXY3p5gOyWOHbtZbLsJthO+aej7DhD+ya/8Ps889+L1H9AD7MwL53jwocf4oR94G9453vym+0rgImxszPloG/jZn3o/L7xwga899Ngt+c7DbofGuV+L7eeCx4tmXTp20Huqxre80WiXvie20BQuWTN4F8A51IFzSk5FkUImJ6GPGbe7SzufWVKp8cTU04RgWvKyEU3TmBP1nrZtB96/bdthuxcbG5bYzcZXVi5SBmdcJInl4sEZzSKAOB2cXrXqeGvkX+Vuoowa51jzDOWgFSeiiuniqQ7RqBbfGEQUxakSgvUmyWIrAlfSmlPHPd2OuiQ3WsZULIk4lM07gcXMCpFSLA3cci1Tj7XayvbP+xKA6qgfLzesIb/gKcdFoawaDiU3s8+uF9td1/PVrz1y2c/LhXpp/BWwrVfBdnvj2N7Y2EAVYo6DKmZ31eFVCN7z1Okzg3TzZu3Y9gZ//Wc+wLve8UZER2yvuo6dnY43vv5e/ttf/BDPPX+WJ5+6ZX2GDr0dOud+uYTTlaKeNS4yrydS6/PAoHe3qNS07jElsljk6stF5HxAsoLLeGpvE8G5wN7uLsu9pTlQAF8doSWqQhPIaj07tCgMahRjzzfghD5GsmZ7TvMQSc0Wc3u/twhcKYoTmdIZxuWbk9/HmSfjw2OMQzQ/deyomi69Jj7L8tmcexxWM1V1hBqnL26MvDXloWQcHZ1rPcbTnjD1nHrvkRYrcEmCeFcKWRzBmWOPXQfYimm4GasVZNUEwBi/jliZyjfLF+PUoeqQotY5DHYrsb237IZag8u807DtrgHboWB7b5fl7gTb7nLYDpfBdjvBthZsCzknlquOX/mNj/Pu73sje8uO9/7g29ncmLOzu7ypY7qxMeMnPvge3vdDb7fAolCcCPzmb/1HHvrmE7z/vd/Hj3/wPTzx5HOcv7B7U9/3SrJD59yndhDohyX5JNJZj2xGxz7o24tTr/RMNk9GT6QP3tqUqlqkGAAyofFk9UV6F1EV5vM5TdOws3OBVbeinc/Y3dsry97W1AIpDZWjIQRzsozqEqso9TSF05zNZhZlD5G2cdwhBFM1MDoFASQXZ6eVgh6X6kNTsZhGZYyO+z86QdOFi0AQZ1SO6kj7iAK5cL61WZnYte6syZg4N6wIpCh3agVtVtPYT8+fc86SqEXBIVUyWbj12PVGqSS1n7W3+KDhF7JO6RllSndMI19VtbbucjhomYPsZrCdc+YLX7oyd/zcmQs8f+4CG3edvDK2Y8H2Yk7TNuxcvMBqNcE2B2E7XwbbecB2ypl2NhvUOc4JJ45v8tC3nuDBb3yHP/r81zh/8cYcrYhw8sQW3/PW1/KjH3gXr33NXZdg+ztPPMuXH3yU3d0lv/cfPseDX3+UJ76LonY4ZM79II31tfCU04v74H/V0Y1UhzkZ6PqOPGspC35zqAqI0MxaUwSUgpG+XzGfLyBsslqtcMGxubmg73u6zp5LKRF8Q9f3zGezIaE4NZMJhsGpm6Mct9V5i4pdKTyqmvFcm0BpjY5H2WGN8usqoC6Vc+kIWSWKlbJBymM1Is+5SNbKDaUkvlTDSH2UJb6UhJxObj4ipaGrKJXOn3Z3FLEbkgsBciZ4T4qRVR9JvfHrqEDCvFnWyWePEfqwrxPnDgz7vy7hZLjh3W67eWyPEtXz53f48le/dcX3Xdxdcf78DvnU8StjWwq2uxXzxQJOTLAdFvRdT7ey5y7FtimnpibiCkefCra9qdNc5t67TvL//btPA/Dsc2ev7wBO7M5Tx/if/8dfYDZrC6bXsb3qej7xyS+xO1kVfOeJ7742wi/VmL2bssuBf7okH187Ll3HZev+f+Pr63J/UIBoGXyQUqEyLNmoas5ivrGBaxqW/YrlcslyuYf4wHxzs0xvEubzBd1yhWrGOwMbZcUw5Z0R67deq0or9WKRuvX3iDEOvHJ1kIPjFp0chZJEzWpVm5rRFIebQXXsYG0BnGVJS+LTTdoSUF4jeFdLs8bkqxWgaNkPazVgMse6ffWzR/qlHmd0jMA159J/3ro8kpXYWzsITbFE7VUVU/j4eizRNScP1hCu/rNDa7/XSL+qTBiUQofDbgW2v/zgw1dVfNQhJjeE7Y0JthcF2/kgbMs+bLcTbId92Pa88fX3sJi3N3zsRIS77zzB3/3bP8liMSvfP2Ib4OLOkl//rU/ywBe+ccPf82qxQxW5X8kuH+NMZAWsc5TrS3WGzoVotqWlG52niDfJnVok0y4WSGhQlM3tQNvOuHj+gjnaGJkvFjTbDefPXyA4K7E+d/YFTtxxEsmmKqgbXROKIsaF1wtgWM7mREzRNMnl9RRuc61QSUrjJQAs2lY1p5hLf5qURspi/N5ptG/L6FrkZBdiGYOXTJgoNVXq1hN8rkbQAlmAEqnXE1Qv9EohaVmVoGqOveybJhsQEWMczk9KVqA0Rum170311+utces22s16xEI9n/Y6V6orD5F3P8CuB9urruPxJ5656mfee/dxTp3YMlWT82h/GWwfC7SzGRfP7cN223D+3AVCmGD71EHYZoLtnhCaS7CdUo/znvtfcydbWwv2rkOK6JypXTYWM/7ij76H9/7Q97IxnxFT5NuPPc1995xka3NO3ye+9e2n+JVf/xjnL+xc/YO/C+wV49zHe/P+R6cJpzzy61Pnrmq0R1HJUIo5LMLNdH3PYt6YmkOV2HWkFAmLOe1sjnOOjc1NQtOwXC7tM7Pi24atY8fo+552PuPCxQvs7S3ZOtYiAjFFGtes0QdTB2s6Yk/txdL3PfNFg5Y+22uc9TBb1d4/3dc6sqxGW6w58YBqHBx+vejqdKaheROTqLLEkd6VRO7kJunEmSJlSrdM9m167GtDtDG6M0e/6lbDCgO18XB1TJ5R/ibBrMnZem6nXHsdWSJThy6sJRmNwz7cjh2uBdvjuX7k0ad46vTVueOnnjnLg986zdtffxevu/MkIRRsr64D28evFduGu4oF7xxpgm1FiX3PbL7Bj77/3fzr62gz8GMfeDcf+ivvQwRms1kRFtgQkd//2Od58xvu5Wd+6r18+o+/ym//28/Qx9vf3vmw2KFx7tfLQ17tfWNCSsaknBj1kLOgRQMuopbdFyv20Kim3siZbneX2PXMFxv4hcc3gYY52vcWgc9aGyTRBLJvSTmzu7tLO5+zWPhhWVy13pUKmSo8VLUkNP2ojKlR8OWOjxMbRpEzMdoAhFh08TVSllJW7kRJKkO0LoUamsoVq9P0floBWykWKf1CLEdhZaMTBYyMTnjU7DtSn4YbQ67UDBZ51kQcWA8UyyXYvqeY1ot1DliFgTUIkzrkY6Bs8pC7sBv64YjZbyW2nzr9/CV9ZC73OZ/442+yWnbcf+fxEdv+cthuaODy2NbLYbspP2sgoYOUV1WNf9eAYlPF5rPmMjezS+2Hf+BtfPAD7yyjFAvwCrZjSjzx5HP0vQU1n/qjrxw59n12U5y7iDwqIl8WkS+KyOfLY3eIyEdF5Jvl58mrfc612MFgWL/wr+QQRGwyTQjB+oOH2qdFSNlkY6EJxVlpoX0zqY/sXrzI7s4uIqYqaGcLEMdy1VnBTbbioa3tbbaPncDasHalF4dFXt47+t56Z1ReuvbqoDh74+DH8vJp1Ioq09RgztnGpqU66DoPKxSwn1XPHsoYs/qvXpj1uNgm6HDT8W7sDmm8thRSWK0PPjXiZ+0zxsdluMBTqslS6IvjmNJVtr9j1J9zXDtnYzK2jI3DWvq6qoZhug2TfSm88LU4kYPs8GA7T1alJhO9Vksp08dEzBNsJ7WeaimTuim2uUFsm97dCpNKIRsjtsd8jaPrIz/4rrdy/PjmVbddRHjfD72du04dN8pxH7a3FjP+wnvfwbe+/RT//qMPcOHirdHMv5rsViRU/6Kq/oCq/rny9z8APqaqbwM+Vv6+ZrtW6dr+JNNBDa/q6yrPXpf0bQjMZy3Hjh2zvuxtS0zRGhmFgHMNqAyNqlDY29lh9+IOq709fBPY3N7Ge6skVbVIMpWoRcssSsglk29JzKYJg1Od7msqBSEi1TGOTqtGsRY9W0fAlLCf2QZ4D20HKI5CL9WbTymUkZMebx6VJrHISwbnXSPx8X1aErv2uyutYqffNXSrrOdSbVned6M2u35vTdSKmMQu5VT6sRunbqez9LEpr/POUdsmMNknS+46G+pcbkhyDdHyFexQYfvCxR0efOiR69qBxXwftv1lsH2hYrthc/vYdWLbVoMppSEh7ybYDt7T94lf/c2P88k/+jOj9q5i97/mFG996/3UXNF+bKeUefrZW1Pd+mq1l0It83PAvyi//wvg52/0gw66GFQvXbqOzcEu5d0Bq3hMVv3Yr1aDgqDvOhQb/LzYWCBOCE2L8zZTMoTGhhl0HV4c3d6S3CdWvTVMatuWtmmJfRSm/BcAACAASURBVI94Twjt2g1mb28P7/2QOKyl20OUruta7cFBii2dLVloipMa9Q77VRKMtey7TkNiwoUPn6nrskQrTCkdG1nPWQBrEf6UpwcGrXnd1ulr1n6fNA6LZTCylpxAnhRtyQSCOdtNrmlmhNDgnC+9dEYJ5CUYmXzveL5rfkDKhKubcvBTu63Y/tKffYO9vdV1fc/FvcgqAjLBdjPB9qpge1mx3YFww9iWA7CdVZnNWt75vW/g4UdPs1x1iMDrX3v3Zbe7CaGMcrwU206Evb0V33j4yes6Ft9tdrPOXYE/EJE/EZFfKo/do6p1hPnTwD0HvVFEfklEPi8in9/ZV4a8H/g6/G+IC1Ad1RMGusmLC99KSQ7lDClDykqfEsuup48JJ4IXT98ldnb3WC1XtLMWNwtoECvBDtZAK/bWw3x3d4fV+fPQ9zgyPjhm87YUOkVms3bQhYOWcnzPatXhvDUB62MkYZGQ9egIqEqVdltCc4ioYxH1ZVK2rpaaI5ISAQg1knUOlbHnyvRmoVkhY0OytczQnHDTCSUzSVb6MKppnIKYKmYMuCzarhH3/gEfVrAkQ95ANdtIN9OtkXL93aNkgohVz+JwTYtOhoADRULpcS7YdzrrsxNC2HcjKt0HNVsJlihRbrh12KHC9qrrOf3M9XdOfODLj/IP/9V/4pOf/ybNrLkytnem2NbrwHaP9wFfsB0taWQFUAXbinDXqRM88p2n2d7a4Efe905++i/9MFubi9LnaN3e8sb78MqB2M6qfPpzD7JcfXc0ALtRu9mE6gdV9UkRuRv4qIh8ffqkqqqIHBg2qeqHgQ8DvPb+u6+gBrM7dilXZBqF7adjhqhWR951qrDIGRCrEu1jxGlC2pZFa71NujJBJqsQmsaKiPqOmK1PhyRP0wS6ruPC+XNsHj9mpe7OHGzXrZjNZrTtzLjGEtk0TYsqdF1XOH9LCLVtS062zPQhgJuMhBOTTdoSuKgRXImsxZbHrqhBxLn1lcol0V8qEkprBqbZGqo5sUra9R41Y4J0qqSZ+qTKkVsytgx/OJByKLNc8+jUTJpnzt8aszkSseQGLFLLKQ1dPNc59dGaps5tTWsR/VRhY0naG4b4LcL2PTeB7ZEq++znvsKjj52+3Cdd0S7srPjDB77B2994LyePbV+K7ZgQX7C96rhw7hxbJ64H22ozVEND4z19wbYmo3ROP/sifUq8cPYCMWXuOLnNf/bXPoh3wv/693+BPmU+/ccPDq0Bzl/Y4U++9E1+/C+8i7vuPGG5r33YXq567r/vTgCeeOr5Gzour3a7Keeuqk+Wn8+KyG8B7wOeEZH7VPW0iNwHXFdp2JUUBfud1uWUFFzy2MTxY/y7lNmhfd8RxBFmDU1rMi8lk2KhRJxjtrFgtVxaC1pVFvMFXew59+JZNo8fK/p0i2y6rqNtZ4BRJjGNMsTa6THnPBQs1a6PKdssyhjjGlVSi3pqZz3nHAmrYnVV+TOJ0mHUhk9pljqaT2R0Gqi3zpEplc6PtdfMusRxqqgph9MqVLGIurZ+rc/XApdcWyX4youbI6sJWucEFUfOEEKLa4J18VRrSjb0zZncvHOhAMYE7EQuWXr1TBOypsy5fs3MYcP2cnl9dMx+e/sb76FtwqXY3lyw2lta+wdVFgvD9tnrxHbX93a+iow3xkhbsP3U08/zR5//Oj/xwe9nYzFjY2GV2yn1bG/Ncc7z83/1A8MK8FN//FXuvfsk99x9h0lkD8D2X/rR9/A9b7mfX/vNT9zUcXk12w3TMiKyKSLb9Xfgp4CvAL8D/GJ52S8C/+amtnDfBbEf9PsjVuttMl7cV/ppv5u+fG9vrzQF28VJKdRRJYrS50Qzmw3OZtWtECc0bcOLL7xg1XveGoX5EIiltakrlaCr1dISj1qGDDtrk1t7Zls7BBm4y9Kvq/CLjtj35DJoJKdIih2pX41NwXRUJEwljkavl7YBE166OsQYY+ktYkVG3rlhGTxt+jU556UnjMd6ztQJTusDuAc6xdXB2ZUz16LEGdU4NvHK4Zo65SeXgiw7vzHGEp0D2BBv3zSlS4EMn2VOZlRnHKTjv1Y7HNge80YxppsuzPnjLz/KL//uZ3nkiefsxj/F9nw2OOabw3bBsoKfYPsH3/02jm1v8M//5UfZ2V3ivaPrems33Hfk1K9h+0fe+z288+2vH4oO5QBs7+zs8ZkHHuTMixdu6ri8mu1mIvd7gN8qF04Afk1Vf09EHgB+Q0T+HvAY8LduZgMvF+scFN1Muff9ydX9xS0i4Lwgkox3jIpqJPVLyBHxlnSyz7INCW1jBUOarUQzC5ubm6yWKza3t0i5JzR+oFNCaIakUwW6PW5zUDVHwBGzDhNsasIz5W6o0uxTJkuyilRxFhn1ER9q35lSPOI9qc+l9Nw22joFj859UKhI6YiZxoZqtehJ/Hrx0rSnyzRxqnlfv5pio7O16L52qXRFemqLp1yaqDlyLsnalAr/CyB03YqcEqFE6m3bllVOZloRm1I/WZkYpKuWviaOr9MOFbYff+IZHvvOjVEyU3vsqTP8s9/+DH/1R97J+77/zeW7bEPCrCHHa8V2HGo21rHd0YQ5KkLOESnYzjnz1NNn2Nld0gTPX/vp97O1taDvugOw7Yl9JBZsf+Xrj5FiZmNjxltefx/Be04/e4Z/+wef40//7OGbPiavZrth566q3wbec8DjZ4C/fDMbNfmsye+Xp2KqVLAuzTUfzD9XKR8lclEBgvW+EFV8Mue+Sol2vok6hdqDBUjJJI2p66zFqbOCjNA0pKw0TctyucdiMTceU6ywZrFYrFFEq9WS0DaleZjJBKXQMlISR9bMq2i7XeWnhSylX4tYuTcia4lFV2vxy1Bjk6SlkoTMNhGn0jUiNqFe3NCl0XmHKz3bq60rVYz/H0/MZRQ6WCthXOHZxSNSxuMVPtwh9GrSS01WLWxzPWG1Z3UF3ntbVQQ/8OhgKhmyJWtdSd7V7ayOfaqguU7c3WZsj/g9d/4iH/mDzxildQusj4mPfOorHNua87Y33buG7Qu7K+aNAzKhvTFsL1dLmrbFF2xbHsvx8z/zF/hH//zf0fc9v//xP+EXfu7HSsvhg7HdF2x/6rNf5dHHrd3Cf/U3/zL33n0Hv3vk2K/JDk2FKlzKO+57dnh87JA3rmz3c+z7k1P2s/wupqRIWQlqScXGe5owQzTTLfdIKdPOtywZWQb5umAqkaZt2bm4g2pxUF1PVGVjY8MGGcTIbDZDtXLidmMJwZMqkFNCmgYR4y9TmgzscAJqfTlMYROIsbcCntojRoAouDJ0uFIR2dnNwhQtpfheakMwAbEpRa6MoZvSOePSt9Ibl7YXsL+VlHSIsKvDBrvR9H0/RHVVDuKcyTstCTyeS+88ilUMVyld1y0RcaVLpSVaczm/4qXw/UrSuk2udNCMiJiMLvbWL9+O1fVz7rfabhTbMSb29m6u5/l+iynzh5//BqrKfNFy96lj/N6nvsqjTzzPB3/4e/j+t913GWy3V8D2OHErpTTQi7nMTDhxfIu7Tx3nyaef56FvPs63Hz3N299y/xWw3VsHUZSzZarU//VPfoeTx7c4e/6od8y12KFy7pezqXMGhqrG6UUw8O9rUXseuMukJvfDyoKqYtIq/kqr206hnW/gEFZ7S/Z2zhHmm9ayQB0igZiV+WyDEyfv4OLFi4Dx5FksIp/P58QY6fvIYrEYdMB1W9u2LX1QShOxpsWFBvp+uDimScGUM3hP6nrEjclQcUYrURzdcFF4yrT7Qltk6xxpKxYtqgMpMzkpN61arWuBOGXakxPKkRq1+LV/i0XobnIOxHrRqH2INWhUMqnQMSOPjxNyn+yCFiULqLP3x95uCKFpy2pCyrg9k71JuenZbpgsUjWZlLLkBDQngrgitzwcLX8vZ1fD9vnzO/up+VtiTzxzll/9yAN47/iL730bMSZePL/Lsks3iG1dw3Yfe5pmRhusRfCpO7b56b/853jo4Sf5wJ97B4vFrGA7HIjtZ54/yypmLu6s39hePHfwQPAju9QOvXPfr4S5VB1z+arUtX+DQ6I0m6qURXl/ie5smotnvthg2fcmgYzRmo4JzDc26eKKtp2xub1JH3syGSeW2FutVsO4seVyWaIc+46dnR0WiwWz+dzoD+esgq/xQ+QzrSyNMU4kh0qMGSgKkazIrC3UTeHHs1Ec3nkr3ikJVQue12mqSp/U1rBD4rH48jEqH5PTxm8nVCc95Cecf03SulLJmBVqK96ctSgkytxU71ESmmyoSXaZ2JmaaDafm5oDJcdIFyO+0E8pmRMfEt7dCsHaO+RsnwHgQ7BEsr/+hOrLZdeC7a88+O0bopau1VLK/IfPPjT8/Sdf+TYheH7sfd9L1nQVbM/3YXuD+XxWlFyutPTwNOW8/cC730IInlMnt9nYmF0W288+c5YP/+on2LnOgq0jW7dD2c99avWyvNqFcLW/p583rNTF5oSuJQlxRBX6rLSzeRl7l4ceKSlGnPdWyeds6IEWZqIm/Sot0TTNkGyazWZj5FPK8KtznSo8agJwf7dGq/rLg3PMKVvTpGwDpqf7F7wfBn6sKVfc2EDsoArU2ghseI7q1CeJ6Ykzr8e38vFT5VIqpe3DcOSakJ3eSEol66BucjK0PTbazFZczazFhVDGGtp78A4nivemtEx9R1wt0ZRrzZUlhW8KfS+tXSu2X0574dwO/+HTX+b0mXM8f26HPqc1bMc+Fmy3RhsO2F4QY0/X9aRS97Af28F73vOut3DyxPYVsb25MeP4scXLvu+vNjsczl25KpDH5f/BKpnLv2esxByG55ZEoykSC89Z+ph731jWPjQ2uFpMFta2LSkldnd2ULVmXDFFfPA0ZSJMdYxTBw8MdMtisSjysRV7e3uld8e6pLNSMvuVKaHw0bk8l3IZIE25WdX9zbq2LdWDVBnjyJuvR/HV1uSiE8XM2NLhYEdeX1OdeEoW8dUmZLYiGiO1Sj9Z9aGtDyyhPG6z957ZbFYSpoKNAXQ4F/AuIKLDmL6c0kBfDOc05+F83z67upNex/ZYlBdjui1VmH1MfOIzX+Wf/cZ/5NkzFyd48jRtQ9/3hH3Y3hiwvWRvb4+u7ypfuoZtDsR2WMP2rPXce9fxl32/X212OJz7xC6JYtafnTx/ZQe//lyZCsT4OzrpRy7jhSXOEdoZqz7iQsAFkyjW5Wjf9excOF9ZaFbLJXUSUc42hEJVh6i90hld15FSYrFYsCgjy1arlZVrFzlZdaxTBzgoRpowqGJyztbOVoryRHUoXp86ZxXIMpbyT+WK9aKcflctrrrU+Y/J1Gmx0vQ1UzMulhKxu4GqaUIwaWNx6pR9894P/etrstU7PwwHqc3WbMVRnEqeTJoKoUwBMv2183ZT8SaLur2+fWLXi+0nTz/HI48+9TJvpdmXv/4dzp7b4ZOf/Rrnzu8WbC9RhbZpSTkNldJd1xFTZGOxwWKxUbDd0cVorTb2YTvvw7Yv2K5Dv0Vhc2N+W/b71WSHzrkfZENEpuuPXS6KP9Dhy6gWkZqAHCiLMoA5W9GMOMdiY5O+74fCm+VySWgaNjc38c6x3NtFVIeCm9lsNkxYSimV95rzqY+rKqvSuCwUuWGlPKZ2UAGR9x4fgrUqhqGIBGSI1oeh0owCEccoUdxPo5QjufY908hx6lin/PpBdQP23nG8n3PWVz72YxVjSnno7CiuqGZkvODrRgfnhwKtlCLeuVI9PEotvTeH3jQznA+oOOuJU8YYOu/x4sr80MNrl8N2jIkHPv/V20LNTO0rDz3Oh3/tYzz+1BlSsvPR9eaEFYbJS6qwXMN2viy2p0EIgPOepPCpzz2EAGfO7vClB7/zsu7nq9EOhXOvkeX+qEaL8oLSUAmqkymRaP1HbUBr6o76b1zujhePk3HIBGLDKFTFhlt4yDlaMUYb2D52Au8DMSZSUrquJ4PpeH1TnFUgdrFQNWGojMyaiTkNevrQmpzShwbnG5x42ra1VrZllqpto0W70wpL1bENr3ehTEhy5tQm8sZp/0Pr/V0/zypKKTcei5oUNKExjZSOKqQ6l7WscBJosoT0WtWrs59Zk31WtnxEjgmHIzhHTnHSbMxeb1SPdYOsUksBfJEzOnGkcuxwQmhb/KwluzLar9z4NGc02T9RxTtH8AEXWsS3qAuoC4gvVM9ttBvB9pkXzvLIDfaSudV25sWL/MNf/igP/Nm3gYptG1Vp2G6vG9sCpVLbsP3k6Rf58tcf54WzO8xLi4Ijuzk7dGqZgQoATOY3Pn5JVF7plfLPHlunE+z3ovooycGUMl788OmqQsYmMLkyT3S1WrGxsVmoAYBEX6LSpm1RTGkgLg9NlKQoOrz3ZSla6IM+2/Qj700mWPpNGS8fzClV5UyYNusaI2IpUsJBxFKImMpd1nmlU0cyRLr1GJaEJvXtRZFSJxYN0VR5/7SneI2qa9+YkR/O1C6GtTjKOz/kA9bPLWtUkBPjYGOZoGPfw3CjrKXsKUVctMEpKSZy35dkbT0WZZfK3+LG1Q9ldXAY7MrYHlv7qirLVXfbo/ap5ZzZ3esJoYVSDTxiu92HbS3YVprCp+/HNjLO1BUcb3nTfdxxYpu9Zcfdd53g+LENnn7u7O3c5Ve8HTrnPq0qrI75Uk59dDolkGSoW7oM9bD/CtecyVkIzibNOIWsCYUhal72HbN2br2t544YE13XVS9VKApF3EhfVCdtRRxC41wp2U5AIvgxIg/NjBjj0FHPpILRtiv4QTrYhEDG1CP1OLji0H1T6J0SWVcHUX+v11KlUnwZsWectRLKCLPhFqj1Jji4HuNB/aVU0Xi+ICdr0GUzM6uqZuT2bZ+LYxctjztS4WRhPIZN6VqYU7bh332i39mzPjjZVgkCpKBFFWQrNKEoaRipI+PoD4d3vx5s/+kXHzrwM15uExHe9qZ7+ca3T/MH//GL5BR58xtfw7u+9w2XYLsZsO2HVgWrbkkTzPlPsV07o4YQUJSnn32B7a0Zr7vv1JBrObKbs0Pn3IfIfPh7XXK3HrkztPWtUfz4PKylrKYRnlhxjDXJMr20c54gxhXGmJjNZ8SYWHUr428xLtcXzW7TtqVlrhVtzGazwYm1bTtQQSKeEGQYhbfsl3jvbbZkHjscpqSQMi64QVFQdd1WIq6FhzbHbcMsSsOx0nzLbnQlIZeTrWhE6AvvHSbdFGvwL/uOy/QcmPNheNF+pc3oqPLws56vKT8+TdrCWKhTbxxQzoH3Q2/vnJI1Det6VsslJJutWrXr3jmsjcx4o7XEdCnwKqsoV/rkHAa7MraneY5M31268nm5LQTPm153F+95xxt58ukXuPvUCba3t0gpH4jtvX7PVqLBqJihe2fKkOQSbEvBdkZ58exFLlzYwwdPH/WWtVv4brZD4dxHPfV+xcuUYplK8tYfU0qvmOE90/KbmrFicPD1xgA2KKMtiUkpS0TfusHZ1OV913eIONrZzFoM9D0hBNPB52zOGdNdi/ghcjeqpmU2Y23Enmompt5WApSZk2IOOhdHXh37+D4ZHKkrS9xxX2x/aqRcnak5vFFZk7N1+St3NUTc0Bph3LYxcVrPT8pjq4LpsYfSs6fQXdbAa1TU7I9W7TNHiieVgqa6T74oklJVHuXMbDEv22qtIoSiiS8n1eivzKwdh0nkQlO5EG67b782bOcB28+fOctjjz99OzZ1zX7o3W/ib/z0+xCEd7/jDbRNYGNjQdO0a9ieF2yPN61MSn2ZdFb69Be5bp1qeyC2y3P/6nc+zbe/88zLvr+vNjsUzv1qdtASrcrmsqxztcro8PddTlSnUiN3ERs0kdXRhNlAzzondL1JGgOFJvDBhg3HaEVCZUlKmd5Ul9gxmt66qgasV0o3yL4GtUfwzJs5KaWJTjzRpUxbqlqnVsfb1X0vqvW1VYy1y81DonKseB3b4jJE1dOZqOsKhtFplxtqkYjWYzc4z/IdzsnQM6fyY3Xzp4M/pjflevf1obYsKDyuji1eNzY3h892jP3pYTJ3lpokVpu/ih0L8Ra1iz88tMxBpnopUh97/OlLVCa3w5559ix7y57tzQWLgt2+tMkIoSnYXiFic2urxDEET7MP2yknUspjxfbke2oPoJxt1bLq4lHkfgvs8Dj3ad6rXPjTyPughKqWx6tqpg6MnvLv69+RyzBfT9t6GsfQhjZrJvjGmlTlSCMzUo6IOJuSJI6mnVmJexm6YbRJ5MKFjo2NzdLlsfSySWmQT9aK0ClN0Xc9e7GzIqgmlOhai9O1yThTGmNsumXThTTXro9uTfvuvAxRf23NaiobW/46sYKo5XJVHLZVhbLPsRcXbadjHy0z1e+jtSUBhRaDrLF8Xk34uuFGUq/qKol0Wrj/WRjaAIvTode99QWaRPs6FsEgjpQts+oUSyhT2gEX/Twln3Bb7TqwnXPm0dukbd9vjz35PP/H//O7/PWffC8/+K43DTgzbK8ITVjDthMpDr8nxg4fHG0TyFlwUgKQnOymWzCVCrZFhO88eYaPfOKLR/1jbpEdGucuk/8z3Nn3O/R1nlJUccWRa7bfNYPigDTOXSxLeS9qCdQyXKA67tr/vE5nqq1vxYYgEZMSwvS5MiavaWxGZO7pV0s0N0OXRnFCcKEkAQvHCMRBL16SrzHbTFRXnWgmZcW7ytkXXj1mYh/NmZeGYSlnXPBlZF6p4lQlJiuKsuEbRc9fireathkrW52QyLSuBdYjY6VMNnKubEct22JYkotYkZBmRbT2mqnyvnpzMM51ekOoy3HrEzM2PVOx74l9b7LKsj1NCOUcClJpJbCK4uDqVjE0jqP0pg+BpCMVdrvserD9zLMv8PiTh4eS2Nld8pv/7jM8/J3T/Pxfed8gJghhP7aFLiW63jpzPvr4s2xuLnj9/XcVbIPlgiwnUrHtEJ557hxf+vLDpJz5zJ9883bv8qvGDo1zRyYBN2NTqqkz38+/H/g4YMzemGQFi4qtWGlMvKakQ3TnnMNL6Tgo4IIVDWm2zpE2L7IZKBUE+mSP+eDpupUNNEhNqfK0QcdO7Dtzsjmh4hwhmKKm75a2jPVNTQEQmmAqEe0RSQQfCM4TgRj7IrOM5ujEoznZ96HklOhTqbQdqJCiLEmZpm2Aehzq4OlRkoZSiosY1DfoeuReVw85Z+pEbym5gPE8lOdFQCPO+bV+NkNl6dAeYTx/MUabZs6oWrKeNK7cxCY3mpJLsAjfislybSzmXKHoDkE+9YrYzmsYfvKpZ0np9lMyU+tj4gtffoRvPGwrCgV++F1v5uSJLV5zz0lOP3eW73nTa0hZ+ce/+gcmm1x2vOcdb+RvfugDRmOKFGynEuU7Wyk7zzPPvcinH3jw9u7kq9Cu6txF5J8CHwKeVdV3lcfuAH4deCPwKPC3VPVFsavu/wR+BtgF/mtV/dNr2RAd/jdeCdWxT9ugritmGDj2Ss1YN8LR0ZgqxPjp6th8cVJTbtra48ZCX5RtKj3Fq4OLMZY2BM7KTsTmogo2xMPaofa40jTMUT5LZHCwFvGCD0YF5ZRYZqVtLFErufRmKSPkuhhNmeOwiUNNsA6IKQKh3KTS6OQ0jUVadT/KisNX7XjZhnrQtaxijNIqCiBl6L1ej3dTEqWDwmUtQTwqbPIQPRfdvwtDTmTa9sAqU/NAx5TUuil/XOHfRZDgS58YhiW93TwsT1EpH9VkIxJDgCK3VHd5z37YsH3x4i5f/LPDGbn2MXH2/O7w98c/8xUAFvOWvWXH1sYc7x3nLoyv+fyfPczDjz3Nj/357+XH3/eONWyr86Rog7TvOLHJ5saMnd2jLpC30q6ldO+fAz+977F/AHxMVd8GfKz8DfBXgbeVf78E/KNr3ZCBQ8+CXcc16XdwxA7FsVOqUqtTnzgjVIeobojutFYz+iHBWTsiWjLUovngrRLUmon54V9NUtZIFIpSIBmFUXXuq9WK1XJJ6ntrW7taEfuO2Hd0qxV7O7us9paQFYey3LvIhfNnOXf2BWK3RHJGcoacWO7uEvuOpkyMquUvNVlVe9PYtunA7U/bBIRgKwJrWVBL/hNSo0cKXQTUBmPT4y7OWgbUHt41as/J5sKOnSHNeYfSL4RayJXHmaBucpw0a6mmtcZtwQdCmadabwK2yhjVQWNCmSFR7bwMElPv7aaBM9394cX2+sSwMy+e48wL5671Yw+F7S1N7XVxd7nm2M2UF89d5Ny58xNsr9awvbe7wz2ntrjz5LGXf+Nf5XbVyF1VPykib9z38M8BP1F+/xfAHwL/S3n8l9UQ+1kROSFlWvwVvwOGyE2Ztg5Yd+yTbbosNWPtCMwBanGEikLp3z6Wucvg6Goi1NSBGVccEApJ+4HamQ5i7kvf8CYEkghaOWL7qrL8TCXCNssplQSk0seeFBNabh7trLFxfxLZuXiRtp2xsbnBfGOTvu/I0XTrvjYzE9CcSFHLykCHKtX91IcPpd2vL7LBVLdVSdh4vyEBXf5JidqHhmZqFbWWBC031JwHPr2eFykl5zUatVzAqJpZV/FMe8X44RjpJLKvry/Lp9HBU0cQ2li4rFYxGbCCqozNgx3rc28PtismL8X2paMgn3/+lePY3/Cakyxmzch51VVIztx/zwka77jr5BaIcOqObbouXhHbf/tn38vv/6cv86WvPT58x21PhL/C7UY593smoH4aGygMcD/w+OR1T5THrqFJRo3KMGogj4UMl+PXp9HgOu8+aqEFsSCzRJ8hePwgg8xrzb5MuliTsND4QOPXnWV9rYQSxWbF4cAHVBNMNOWpNlgq25dTLk2XoO8TF87vcOHCrkU/LrOYt9xzz50cP75F7Hu6bsVWSswWC1arFbHr8cHTzmd47+lTxJd+NtXx1qEVVc0zFhFlmmY2ONjKnWuhhoaCqMmqR6b8tmppX+BKtJ6H2ZfTHFqecQAAIABJREFU4wOFIy9dHqfDrIczXSPyA96HWL1Bdfb1vMayT0b5lJUE5WZQVh3eOXLf0/URP1swtg++LtL9lmK7koNXw3bOmYe++dj1bOdtsxPbC372R7+P7c3ZUESYs3WHrNg+f/4i5868wO5ex9NPPsli0XLPPXdx4vgWsevpVuvYXjTCh37i3fz4n3+7raJT5LNffITPffHbt3t3X7F20wlVVVWpzVKuw0Tkl7DlLcePb1GLUSpnPvn8gVNPVQ2RdWzlamH/0FumqLELdTFRyXhH64TGO2ZtY60GJvSFvcsBjtAEm5fq3RBZBzFly2rVUaNI19hQDFGL5L1zeBpWy5XROc4StClFiNCtVnQx0qXAU889zZnnzpKy0rbCRtuye2GXh8+/wNbWce656xTHjy1I3R7HTtyBE0+felKOaI7MZ61VCqYMySL2pmlAx+pBVS2rk5HPVhVwDdY5Vyd5AGs+dslKqd4Ycdg07XJTjamcC/t+JyOtbJ+brChL8qBF14nThvXWBAUTg97Zkqd2c649ZgC8C2RRcuqQkmBVofQXSja43Ld4PwPcsOq6EXtpsT0mobMqv/fRP+LpZ87c8La+nHb2wh4Xd1dsb7Rr2F7F3rD97GnOPHeWWLC92bbsnt/l4XMF23ef4sSxDcP2yRHbToRT2y3zWUufMh/68Xfz/ve8mbPnd/mV3/6jtdX7kV3dbtS5P1OXpCJyH/BsefxJ4HWT1722PHaJqeqHgQ8D3P+au7U8Vk7gwbRLTdih4zRUJlH7/t9Fxpa3bQjMgid4h+ndfcndTScSeVzRbRmd4SfdGBtohdl8Towri8x1pCOc92UwNYDSUarxUjT1S2dSxlWMPP3sRV548QIbm57FLBAE0EAflT5nzl3YYW9vj1Mnt7nrzjsQaVhsbkBOpKw2xBvr1W6l4Jm2nVkU6zJeTNpoEbo5ON/4IXo06aGsRY+WjB6HFE9XKloVJ5pJ0aSWdWzhMLNWsTm1GouiqBlqCGpDMaaceZVSMlI29XeEMtjBjqWtutzwXbnvEbHP7GNEyTRejI7RjG8DPjQkEbh+KeTLhO31VhpnXjxP39/+lgPXap/50mP87I9+L04PwPYLBdvzgu18MLbvvuuUYXtrgu08YhuEe+/cYtYGTp3c5PkXjvTv12M32gv1d4BfLL//IvBvJo//XTF7P3DuWjhJOCBZesDva1xlcSj7aRlBSpLQdO9eoA2expsm2lXaBWibOYIrMyKtNWwdVrG/cKdWtc7aho2NDRabG2xubrLY2KCdz+3vrU1rMjabGXUSWpwLOPGsVj1dlzl7YcX5C+c5dWKLUycCxzZ6ZkFREuoFvEe8Y9lHTj/zIo8/+TwvvHiWbm/XCn5iR07WHTH1ViySYk/OkZj6IhssiUbnERzirOVvOWRlv9zAdatiq6Fi+3v4WGl8tFVIjMPxrauonDN93xOjTZYaWgPb2QMximzt5ssoY6zHd5pshVpgxlAdq6poygTnSX1ib29J31uhmZbqRnzANy3ZjcO3r9NedmyfPv08zz//yuqA+K3vPMevfeRPeeb5i6xWPasuGbbPn+fUyS1OnZxgWw7G9neeeG7Edk7k/mBsn9ie8V/+3Ps5cTR677rsWqSQ/xJLMN0pIk8A/xvwvwO/ISJ/D3gM+Fvl5R/BpGLfwuRi/801b0n1PPsczTQSt5/Tx3X97cNFkwkieKc03tEGj/fWi0QLpVKjSGvEZQMv1GVril4dO6MWXCUP0Wkd7iHOGliFyTamGIl9z15Y4l2DD4HYZXAdqz5x7kLHsWPCHceVdhboOkV8Q+MVl5UmwUohJSH9/+y9a5Bl2XXX+Vt773POvTczK6uqq7ulVnerJbUkS/JDMg7ZgMGAwTCEBxsiYHgEg4EY5gMTExMxEzMMX2a+EEHEvIIPQIQJGMYzwzAmeNvGgO0xlmyBH5JlWW+pW+pXVdc7qyrz3nvO3nvNh7X3uedmVXeXWo/KtnJ1ZHTWzZMnz7ln3bXX/q//+i913Di4Q8w9wcG5/TOAZc7O2aPLmsjOMfS9LUxOyCGj3sTQTB+nQjIb9LlONJrudI2JA7HohPgyRKO+r1MYRbPx9lO2D2MtpNbOUOe8IWeaDftWxgA+zdxFpkFcyrMvxfGS8W8pbsbE4dEhsV/SzmYm2Kagqcc7j7oGKbUTwSGvEt2/Eb4to3My8e1t6Gu97un74ZVOcWLtyo1DLt9Ysv/mGeshcnBrMN8+q3RdYH2fvu0dnD/u21KkQcpi/+hDO3zwO97Gv/3Qp04Lrfdp98OW+VOv8KPvv8exCvzl13MhtRFmk/Ade4Q1FRzZEscPqf/ICEoTHF3wdI0bO1ARa2bSAlfEOBgana0QKWyggZxt8LTpV5fzC3hvU4CAsflJnMOo1or3Ce8anGvwbkXbtPTrxHqlrG72kDPnzmbOnRO8X3BnmRgi+NWKGDNHK9NgmXWBmIwbf/vOkueev0TbdszmLXVrDxYsUrShIF1n0AxjRg05V6ldvxlETd2RuK1z1XuvGXR5oXD0Yxm+YfeaS4dpTokmNIavO1foh0XgbIxjBpFl3bBjir+UndSmKFez+VqArnNXycZpjynSdR2LRVekCDbUyJwU8XYdJiwm6KsE92+EbxuQeNy3v9KznFz7yG9+mTc9dJ5VNAjs3LnM+Vfw7cOlJQt3+fZzF+mevodv67Zvf+f7nuTnfukzxBPW5HVS7URMYoLtrSqTh3v3V54cc/enRAS8V8vagxA89uUc3m+2/Fmt9d3+WQY/u8qVDiNWX+EYgwsoBVLrgqxcdx88bdeajsz4FejmHaFtaGZzkm85XK/Y3U3snwnszGeEpqGdefYWwvk9x+5Ow82jzM3DyM68ITQZcQnFcePWkpcuXbWsWu06VBNIJmsk54hRL/MWXGXwiy+a6Rt8fTo2b4sRo5tAWzntOaeRxplTIk/mvhrVsTwH2bBial+B6eWk8j0j7/w4RLHhyd9jhF+BgJxzdPMZvm0Mbql1kjKWzyZhdVaEFVd0dx50e+qr+3bOmU995tkHeHVfne3tzNk/d2707bP38O2H9hx7Ow03l5kbdyI7i7t9+8VLVxiO+zYFDkzVtx+8lMQbyU5GcJ/E6PFDP/1ZxdjvOnYDz9T/mzCWI3hB84CmAS3OUXkavmjH2DnymIGK6kifrM03cYikaAHT+2CdpFUnnE1BtjYwhTLrtJ3NWOzs0M3niG9Y9pH1sOTsvrCzmOF8i2s8bavszZSzO575ouXq7QFcy7xrWOwEZotAExpa3/LSpSvcOerLW1IHctRpS6VAqqWFv2TFI+UwbUTLBMZgX+8BNoF08uai2aR8U9HszimRyuSk2uw0wj9lFJKIKwF3M7EpqbGGZJQbsK+qBzOlR45F7qITK2V3ROHeq6jtmsQE0Qx6CjRtRygTr7xzYyfyV8aE/PrZ6Kts+/L1G28cfvtx29tdcNQPrPtX9+3ZouXqrYlv7wZmOxPfvjjxbWp/xGanl7Oyu+j4wPuePCmP88TbyQjuTDKZ2uhRtUsqXpnLbM9JdimqtvVWRURxmgkCQTzB2RxPLxsluxTj6BgbSVHrwxdVchxIw2A8eCCTCjPGON1pyEX3xd64MXO3PN8Cig8Gn3RzQtMx39llsbdb5FAzi0VD282JIiiZEBx405A5TJHbdzJn5nD+whmefu+38+Yn38GiFbqQSBEuXTqwOoEHzY6sQlZPwtr1U1bbtlZM2ym5vKsVlqkdulVadwsqgZFmqjlbV20/IHg0GVSSVclJC1IlhaZq+jmpXzP0vc2VTZgAmKaxgzjFYexqrVTKypkfu1+xhaYO2xDv8E0oGjrgcEgyDqTz3vTCuznSLWybJptA+jqYjF9zezXf/vKXL3Lj5u0HfYmv297/be8afXtn0dDO5kR3t28fVd9ewPkL+zz93u/gsSef3vbtizfL4BxQdeQsZDxJzbe99/zA937r1uyBU3tlOyHB/e5ta8W/p1v2u35L2eDIFT4R47XXDKlqu1Q1xypitWHDUPDcXLjxRWa2NN+AEgsbJedISpGU06hrLSUz3Axb2HRfeu/xZVxeTdMaHxBsqIVzUtqwhcHBtYNMisKZswNPvu0Rft/v/WF++Id/hNmZDj+zcXtXLl/lcLkCBC3t9apqxcRJkJZCJfM+WBNXCeIxZ+Ozi4zDxVXswzMunDAuiMMQi3zwMC6QmgecZNBEHHrWR4fcPrjK8vAWw+qQuD5k6A9J8QhNA7FPaHTkqGWsYM8wDKXpZdLkdY+mtJrFU3j0LpQFtQk2PDtsgn+9x+2vr5GLfhX2Sr6dc+YXfunXWZUW/jei/dKv/OZ4X40PiN7bt68W394/O/DWtz/C7//+P8of/WN/gdl+8e0mcOXKVQ6PVrZDk3v79ic//+KJE1Y7qXZyVCFh/DAe/4BufnbstXGLWyEZCE5KQ5Plq1EVP5kMVId0OArXO+VxYcgkGt+Z6mOBObQ0O4HiXGPNKDnjygi8vu9NTKx0v6ayu3De4RvLjNs20DQ2DLqqRPoyhCJ4h4uegcyN20rXNDx0tuXKpef5Vz/5f9J2Z2naQFw3OJ85XA7cvHmH82fP4Z0jplQy8CKD62ysnDi/KTpLeYtkoxdT4Y+pZa1MIkgxEWMa9WpGFUjJiBOGlBj6gdhH+38c0DKXtWkbutmMtpvhg4mniYv4MixZ6jWxYcTUBXf6tbWwK0UmIZuUc60XOG8YbU0QxnPb0PPNfKAHa/eqHw1DnT/6BjZh9G15Fd++Xnz7wtmWK5ee46f+5Y/Rzs7StoHYNDiXuXM0cOPmbc6fO2dCfGXOboVAxXlu3l6OCd2pvbqdmOA+DeL3lBSYPNDjxyjgxGiLtTiqZbYpY2G0FGoSBAkjz9tBaY4BcmaIA86ZkqH4GpzbUTjMAneBNErxFRg14C1Nrri9aVeH4NmZz3Eo/WogL6rEr01ucimRW1geRR4649ifB1zuEX2ZG1depHMObRy3xKZG3bx1aJOIQh3lt5kqn3LGU3VwmNz/NqZdF9Ha0u8cpLJTsaZTg3dqk9R6tSQ4kzxer9YcHR5y88YBBzdvcfvWLdarNThj+gTnOHdunwsPP8zOmR1m85bQOUIzL9x/ZywWJ2N/ga09d1/nGOS1DCzBsvdxYMe4CBcVmcLWyeXeXfGBB2nTutDUbz/3hee4eu2NxW8/bk4cO4vF3b6d7+Hb+8W3U4/oJa5ffoHWOfLUtw+ONr7tZNRKAvPt7/7A0/zir36e4Y2+KH4D7MQEdziW3eS7t+nTrftUUQ/UskaxrMF5w2uD3/5gV5xeVfFebMCElqACUPjYOVs2HMQV2mOlDZqWuzuWZQLGz1U3yUAdlMlKXddx7tw+3gf6dUK0s4WkH5BsOw6ngsuZJ5+cszN3ZJQmCjszhzS2XPibAuJYrQaSKojDty1otgJmbV6qGbK4yfY2l3FmRldEpQy2kFHJYbxmAN0sGimmwjEfuH3rDhcvXuL6tWscHh4S+768nw0uNIg4Uh64/PJFbh3c5E2PvZmz586we2YBWpqqmgbvHbhagC2wWfBbAX3qE7YbKTsNkZLFW9D3bpM5FiX/Mrz8BGR4IyKz7dsXX77Kz3/4Yw/22r4GdnDrDh//9DN0PjC8im/7dLdv784cNLaL9t58e7kabLrWK/h204QHvVa/YexEBPfK/hg/yPeAZu6Z0bOBZqRk7j44vMcKjpotmCDlmEnzUsxbwyos8xZcU/TanYDkkrGHMbhrFtTpGIAqPODGSUplESnriveO+bzj4YcfZn9vn9XRywi7UK4NAEk0eC7sdjzxFqPzDSIEl/DBujE5tJqAqpCyFHZIi8vKMKwN/igQjZTs1bRxGIdKZ1X8eH0bLJtCkpnqvNSBJDlF+lIkvXb5Gs99+TmWy0OcE2at0OzMwYHzgVxojsGB15Y7B4e89MJzOPcUIg1zjSg9bXkOTuxDXbtSj8NEUBcrP07VEkALJW4TvMszEBl9yUYu1nHMD9bu5dvXb9zi8Gj5oC/tq7Zbd4749Bee53d/+5Msjy4jsov1mkx8WzwX9jqerL7ttn378DAXyqxJIjvvX8W3T8ITfWPYiQjusMHbzfnl2Ot613GqoKVxqFCeTdPbiRX7cgVHSmAHyxSzlVmzt0lGIhjlyhkPw3qdCq86NAXzNZy3CaHAO3VK0SarLD1OBd4umH8sr3lh58yCR558gmc/87KxT2YB17Tk/sgyUYm8+0nlTGvbT5saOhBCQ1wDyRaxdQbVpmDQpr9iwTyAyDgHFmr2q3gXGLKWzNykCGDDAa8FMIOyHK5pRz0YAfqoXHnpCl/63DN0OwOPPNLSdA0doK7nKHriEGjmMxIrWvE0znNmx3Ht6h2u3bjOfL5H0w60rV27K7IPNXPHbXPtp5o/1VzZgdixRv+0sW1FVz8bxj7WFRDqhKYHacd9O8bEl5+79ECv6WtpTdvw6Fuf5NnPXCau7+HbHPPtXHy72fh2VvPtTAMqr+LbfixIn9qr2wkL7iVwH6M81p9vQzElbJfM25fMXcqs0MqxpsgGqIDL2Rgz5ctofJOmKE3k3gqsQVpcdOTc07YCGYZBjf3i/Ob6yjxS7z1ZDBSoBaDgq3hWpJsrTz31FBef/SIHtw54eLFHjKDSoGRi6jl3dk5Sx5ANr298Qxog95k4KEOyOZUZkMrEwTFUno+a3vsG86/yvh4nucyLtYYm0TzJlGuQd/gATj3DEPFeOVwnXr54mRe+/FkefWSPcxfOkL2Cy3QZksDzX1qyXK1537s7Zs0O2TUMMdLOPYvZjOcuL1n1PfO8KEHO/p6b7nTqlVSMdbIrytZma4G95u8yllQ2z0Lszl3B/k224MFi7nWhn/r2F555gU986osP7Jq+1ua848kn3srFZ5/h5sFNHrmHb58/NyflY74dN74dc/FtfXXfluGNJ9PwoOxkUCFrJq4bbvLm39wV6EdARhQbwlF45xR4RdyI0aHGm60NPZXip8mKc0Ec3jqWSgfcJqDEWGmANrO0KhWO8gRiuHUdrzdlllRaZEoR7z2zWcejFx7iHW97F1cOImm9Zu4ixupoyNKR/YyBlpQ9WT2aBMnC0Cf6GDjqtbBgFN+Esbmn4vwhhBGWsMy9UgmxZp/C+a/3M4WV/EgTDeV8xjC6cuUGX/7i8zxyvuHhh8Uaq+YdbYd1GfrAtQPl8kFEdWB31jBrHe2Ox3WOxW7g0Yd3WcUe8U2hhwZbhCbyDVNIpk7Gmharp9maQWn1OU+y+3I/VV7Yjv26ee1923Hf7vthazf6RrcXXrrCL/zSr/OOp97F1VtfgW+niW+vtQjcvbpvt03g8ccuPOA7fmPYyQju3IvqeDfmPj1ug7hnHJnGG9YupSHJ2s8pWLjt5CteZ+eoY+k2BdqUIzEOxFSzceug80V8qw7ACGUWKRUHnlD3UtFcMQWyTaAXB7PdwNvf907aM49z+eKSmcLMR1IaUAcqVlTC+XIPNv1ouew5XCl3jjJt0+JEaTqTNggh0HQtzpdr86Ho4ZTFTMUAjIKvjwVLyluIZfZVPdKFgGBNToeHR1y5egMP7MwEEWsXD429rz4YI+loBYl9di58C7Pzj9DNGhZdQzNvaeeOc/sdITgruDpjGoWmGd9L791kgbE6QRpx8zofN9tzrEya2kcw6VnIqht54WL2s5MAy2zqRi9evPJAr+drbTlnPv7pz/Pku99Gs/eE+Tb38G13t28fLXvuLJXbxbdFlKZrCW17T9/e29vlHW9984O+5TeEnYjgruhdQbt+IO/O2stvlC26iBKcEAoOS10E8oYSWeVqoQzILpljzRanHzwrgm7mo44j5GDrOmpAr9llzRZrx2eMib5fFyjJtDJ8C/uPnuEDH/xe1rrP9RtLWqcsWs+scZSGa5wanTH2keUqcbiGG3ciST2Nd+ztzJnNZ/jQ0HataeZ4b408GKtEwaAMFHKZQYoVhceOoPL/Ubveu6Jnb/DBzZu3OVwPhLYOu95BJRibSI1JZD0Byrlzj/KB7/lD/Lbv/T57j7O9PzY8I9M1tamqoQlNyc6LwNdEb2aL5169YJKdj8wa7wvTp2jR12dybAdgWf4DTN/vSlgyl69cf3DX83Wyl16+yv/0t/8ha9njKO5x7Xrx7W7i27rt20fFt29OfPvM7pzZbG4SHl13D9/2v6V2PV9PO4GY+wR7vwdjZmsR0IwvTRNSAv10H25JW2GuVDZLxWhzwXlLdVagCIsZlJNTJrtMKI1IbtJUM4UB6mShCuXknMZgZAuKjeNz3vB9VeGxx86hv+N7+dwnfpF064j93RaXlJQiThtM00y4c3vNzUPl5lHg4GhF085wZPb3dmibluBtjJ4WuVhV3UAchcbpCrxidDIL9sGHAldV3XRXxgvqmOTGIXH7zpKjQXFujfNYkay19w4NpLhiyAoOrrz8HP/fT/8j3vHWHYOBRI0907S4ISPJtLmds0lXvsgKgC08ecJmmnYdTzuJa6NWhZxy6WXwIUDZMdWBHtNCrD7AzP04E+za9QNu3Tk+SPqNbyllvvz8RZ5/8RI/9AMf5ICb5Orb+W7fvj3x7ZvHfbttrV4lQl+ad0ffHnsYTu217MQE92oWQLeHL9fM7HgG7xyjOJQTLXW6QjcrWZLDXle1ievOCaJuDLzAJhCnTFRFgsNpGANELfzJ5Hso2PswjJiwjLrpBiP4YDh3FSZTEYI6Qtvzlnc+ye75HS5+6te4eflLzBctKhBTT0527juHmYPbcOVGxIUFQmbWec6f2x+lcKca69PFazM/dFOkzGrj+HwI5Bg3AbAEVWOfJHJSjo6OGIZMUg9i7B3nheQGNIGqJ6udN3iPkxVtusjq2gxCBySEgPhAQ0aw3UvVe5ey07JFZyTOjbTWevEjrbRSXtkE/NpUNpWnULXCcAihbOLSiWFXXL12k3/2kx/i6Gj1oC/l62Y5Kx/52Bf4A7/vO5jNjrh57blX9O2bt+Dy9YgPCyAz7zwPbfl23PJtEeHg9iEf/Y3PP9ibfIPYyQju98jSsyqpSF5pwdY3GVghOaoFiEwmobgxY1Mq5s1IqTM83rL0jCtdqqiiOeLqopFtPmcsuKAUqKSVDnUOLUVH06wBF0zQyuOtsUgtwJsWeSaLQRo52xg4aR2qDW7IPHT+Amc/+Hu5fvklXnz2Ga5cfJ4U1yRNrAfl4LZy8w403R6NU1o3cO7sgv1ze4anxzSKcI1ccV+mG1lqS8w9bdeNQzeMYeLI5d6c9wWf97gsZLFjDg4OiX1R0swzRFdojGTn8AoDSnKKy479HYeTQNc5VBJOEqiQHXgGJBlNMbSB0HpcGxA/kkaNiik2OcmYM8aMkUoxFUD9FnwDFPrmpiBcd3w5K8Ngi7duQ/APxGot5mO/8Tmu37j1oC/n626XXr7GP/jxn+eJxy7wx3/wg2h/m8v38O0bt6Gd7RFE6apvnz8z8e285dvihfms4z3veisf+dVPPejbPPF2MoI721THmq2PQf9Yl2o9vgZw47krVaq3oDEAY6GzZvPOY9omWilqeVTpy5qsAOscqGMYerIYzidl+980li0rSmga++DGhJIJTTM2DNVC39D3iHOlWCTmrOJovCvDtoVzjzzG2fMP85ZrT3Pt6lVevnQJbh/x0MLxjvdcYLZzhuXtQ5a3b4Gu6boG72WrGDzWCXQj4btercZiVB0fqFnR0pw1wh6VLegw5ccMt28fkbPd89GQSQlIkVY6Yh08oWoTonY9beOYNQ3OeVJyaMJK3TkRlwkItF1jw8e9L8VtYbp5qFl35b5PO1XrjmjK8Mn5GA7v3CiXYNn8Nk/+QZmq8qnPPsuvf+JzD/pSvmGWUuZLz1/m7/yDn+eD738nv+1bv52ugUsXL8LtJQ8thHe852Hmu2dY3jpkefsAHX3bFQG+dMy3lZgSXRt49MJZXn6DjSb8Rtv9jNn7e8APApdV9VvLa/8j8J8Btez/V1X1p8rP/nvgLwIJ+C9V9V/fz4XUD/HIZc/GVx974zc11hK5LbA7B97ZMI4RQ4eyFd9s2afdo5sAYVmqlqwRDMeVDKpG5ZIU0IKtb+RxGWl4G652Jqe42SlgbdVNaEqBNVo2WoqcznkTGkuZlKFp55x/9HH2L7yFp9/zfvr1mj72iCjL9ZL1nTU3rs65fesqofGFkePukmeooWy9XtsDbkyCWCSPgz6apsUmFzN+eKwiWfZISdEMzjXMZh1HriVGJfdrq0GIRyUjKkgeOLsLewuPI5GyYeaiAjni8BzcWTNbnLcCWQg4Pw269nxckUuY4u01iGs5bFpHm+rQ1ITAdi6MWjM2wEReEXP/Rvl2zpmLl65NZKa/eezmwR3+zb/7GL/8sc/znd/xbi5fuc5f+DM/yM5OB6Nvr7hxdc6tg+rbkVfy7UXX8Z/8ke/j+s3b/PLHPstP/Mx/OFWJfAW7H7bM3wf+0D1e/99U9f3lqzr/e4E/Cbyv/M7fEpHXFF/eZsu8EkPGjizgO6BGCfSOUHS+RSygOqvx2WuqmzmoW39jW6OmsJCRwvdu2pa2m9E1LU0Imx3FKOtbcHTNNgykjLujascnkwUeGSBsAlY9R9MEuiYQJJHTYCJj3hE1Ip3QdDbRad4ZO+bM2V262czOkax+UFk6W3o7utFH7/ueFOOoZb+57xosi4ZMZQdpEXwSTyOeedehruVoHYgpkodIKPCIcdzgwvmGtzzaImTWg1FJlYiw5uhoxWrw7J97lPligQuuyAJoKYJvArXAZgteArevQ0C2CuWy5Rv1PTa6qo1M9OMCknmF2A7fAN9G4dqNW3zqs8++5qG/le3mrTv83Id+jU9+9ln++U9/mMvXbxKawKKbM5/POXN2l9l86tvuHr5dk6jM3u6c3/nB9zLr2gd9ayfWXjO4q+ovAPfL3foh4B+q6lpVn8WGCX/wfn7xlQMWlnhoAAAgAElEQVT69CA2gb2wZJxYMKvYuIysC/uVsSBDmcAk2G4gJ1ALzuXAUtwLOB9wLhhunDN5okBXMe7Kna+LSE6m9R7jgKsc/OKYqYykQ03zOjiP5kTse1IaaLygOpDiGh+UrnW0nUe8wSRxMI2UIQ7s7CxISQHPMKQxYx311/uhaNGrwTFi1zwMg02Vj2mywFmWa7KzgviSBWsieIcX6EIgNDvcuJ1Y9rEUP5Pp1kiDSsD7jJeEakBpiDGjSUjRcePGEXvnH2Nnf5921phypkwXdINRUkrElG0ISHludbdUZZrrWEQpaptjwXtcEDZftjMKJvz2CsjMN8K3FeVjH//sG1qz/WtpqvDhj3yc24dHaFKGwYr1o2/H6tvxHr7dk0oC04RA8I5z+7sP+pZOrH01PPf/QkR+Q0T+noicK6+9BXh+cswL5bW7TET+koj8qoj86tFytR3cdQMDW+GUka5YP6hjg1JJ6spn/56LRGW5GFTiSiZf5UwKFKHZEKBsb4uIxxXBMNVM7AeGdW8a7sPAMAxjZ6pl6Hk8n2X1ESemR9M0DQ6I5VjvPU0wnQxVZUhGG1RnwatpGrxrmHdzmqZowQs0bcvemX32z5xFxI8Zzkj9q1AFQtu2ZUqV/V3BmkYExhrGlNZZB1qLE2O1OGU2a5Gc2dndZzl4jobIctWTUm/36zvwM3uvXMswNKzXnpwC62Xg5rXMYnaBhx55nJ39PZoumPJjpZzWp1PxdiozdTMScJPFSxFzq9TGydjASSbvXWCcqypupKo+KN++c7jkk5955vVcw29Zy5r5p//iF7h1+5C28Yho8e2z7O+fxUl4Vd8+uH3EzZt3aJrAO546bWh6JXu9wf1vA+8A3g9cBP6Xr/QEqvqjqvpdqvpdiwI1jBll3UfXwEP50AsgGSThHTTOjZoylJmLdvymwak26oxbuzLsrAaLNnja4Gh9GcvnA158kRwu6oMVjimQhi+qi1WaoMIwMUY0VXhGChOn8K69N/5uaXoSJ7jg6WYzmraj8aEMFTGnDr4peLOScsShzLqWpmkIbTPeR5UTyFlpG4OSKBzydjZjNpsjPqCI8ctDsOUsGaMmFYXFlGx3IC6gGdrG432i9bDoHNKe4fqtGf1aWR2uCDnhJYF4VDpiVnIa8DlDn7lxcISfneeRx9/O/rkztK3NNVVN43hEVweWF0aMExtsXvxjq3hqPPeqr9/RtTNC2NBVq9XGshCspvE66qlfU9/OKbNen+qhTE0VvvjsS6z6npgNLpx1nSUybVO4b8d9u6Pt5oj33D5c8rd+7Cf4m//7T/DhX/7kg76dE2uviy2jqi/X70Xk7wA/Uf75IvDE5NDHy2v3c87x/1tf0/9yBsl4UbyIaclYSLqLPVNhEQvwbvy3ZatamBmbAq5zDlyDUyEmcKojfuucQyvMrJmcBUlpXHzEO1yBcFSt2UbzQMqRprFWasEU8WSSLSfNeOdpmwZfOesU9kfhatfM03uHaCgLlJLSpnEpBJNSNR13T9O0NosSu88mNFvMmLpoCQEJbnyfjTpoz2E+7wjeMZsH5CCzXp3j6os3mPk1Z8+C63paibSYSJdmYbXuWa6FVQ+LMxd4+E2Pcfb8edrO2/sYtvsGigcx5bSjCXWu1AR0pDgWX9vQI491olabisu9HqbM19q3l6v1V3wN3wxWGWVgdSZRT87uvnz7kYfP0Q+RePvo1WHcb3J7XZm7iEz3Qn8U+M3y/b8A/qSIdCLyNuCdwC+/1vmOF1TH1yfFlEqXkWxBy0mlPzJZBO66zq0P+LS1HSqrYgMAWfMECLaIlAms5Xc2EEHFAOvPKNllCI11hVbdGTUoZLVc0pehFlPKR9u0hQO/GTUXmqbwujOCDZQWoG2bkb6YihJlxaVDkSIGW2iyYCMGoylIjlvbkRpoX/V6KqxR8XgRYTafsdhdcPaRM7z9bY/x7ne9jUfe/CSXD+DFq5mr1zO3DhKr2wPDYeLoduLWwcDBncz+w0/x2JNPc/7hC8z3dgizFhf8Zid2rHg+fUa1IWwLVpPtYvT096q+zDTAT5+zFWrvP8h/rX371O5tCtaxjPl27baO9/TtkhQU357vzPizf/wP8D2/7T0ngup6Uu1+qJD/D/B7gAsi8gLwPwC/R0Tejz2jLwH/OYCqflJEfhz4FBCBv6yq9zUPq37gp6yPaRZWMVlTcpQCjYzcSAphbpMHyjSOboLqdtAwxUT724zDr0MIhWGhm6BZtvs1wKdJxl9nmHpvQlg52+xHO0P5uzkTc8YXOGcqO+y8G0XMXOkgXfebAdxK0WTHMtNhGEYhMzBIxftm7OqzexfwFGjCFhA3FihdoeWVMXpYhpQKfBWagGrGa8AFRyvKbH6G/fNnuPry47zwpS9zcHjA4VEkkOlmLc284+EnH+bMuQt0izPMZjN86/CNNRYkwE947ffKuMbg7TYTmerCWt1IVcYpXTpZmKaU11pLqJj8K8X2b5Rvn9rdlnPmVz76Gf7g7/suvAv0vJpv279jiljvm/Cbn32Wn/7ZX3mg93DS7TWDu6r+qXu8/Hdf5fi/Bvy113MxxyGZ46+5AsdYXS1vCqlGaadCMjoG8w1XGrZpdtMi3JjNj0F6k2WOmuCT448HlTrSzQK0fd5ty1kCc8HZQ8kyQwh2fIxjZuK9zQVdr9e0mKPHIYLqCNkAI72yZuF2LYy7Cec9ORaesII6D96Or0VMVdP6qDKqORs/3ZXziBdC40EcIgHJEd9k9tqO3Z3HeeLxR1mtl6ScLGsOAfG+MGxs4Q0hIMFb1ykVItnenU0Lp1uLealVTBUf63tk9QubAbvZ1TEeU/8/PlteMbZ/Q3371LZNRHjTI+fv27dzSsjEt9vyuTmFZV7ZTlSH6nFoZroFp2DCTgyWQaoT6BgQhMqF1Q2LRgRUxg7Net46Zq52tVpRNowsi7GjUwtU4KvaYxwzd8ACZM7gggXXMtHI1grTN0kxERrbDeSUiUSapoFSMAJbDNq2pe97+r4nZzs2JStApmh0xvqeTAOi/TuPGWvNgLp2DmEztWn7/u043zSW6eeMloWvjtgTEbw48tqBGBvIeaBNzJoAYhxjHxqcb/DixsXXu0Au8JbtFGTk2d/r2dd7GruTYfzQV5kB57ZhlikUM9311fNoLayc2okzEeHxxy4c8+36vO/27VR2vTnbsd/33e/j+vVbfPQ3v8DR8rSucS87GcF9QmEEth7sNPMW8tjo4kmgG5kCIdcBP0WUamOVejfN7ExXpuixqxg2nTYDpGsBpw7jqHM7/UTPfcqvroB/HCLiZJxkZIuD0GvGO0cTGobSnBFKc9R0FzBSFbN1WOY0kFIcF60mNKxWS5wzQSzNJeiV9yUOA0dHS0AJLjCs17gQCGV34SZFypTzOEzEArnxQ2s27YNYYbkzNT/NQ1koZ3iMhurFsny7BofzTVHtUxuCYm/4WOQ+vsBUM/hpW2/eoDLrxN1kcnIsCZg0qI1Vl4l08Ot2ylP7eltOiZQG22Fqpgktq9XRPXzbdp1x6Eff3lns8YO//7u5dPUGX3j2pQd9KyfSTkZwn5h9cPP4YVUBURvs7Cu9sXCdnZbm9aIrYxCNjhn3tAiKVuy1dEVSZ286E/wSMc1pMSrl1kDmEoicc4SCmQPgfZECtkEf3jkbwYfSeMOrbfpRaZbKynptEgA5JXIYmC8WDNEamqRIKHjnWK+XVgOImRS1aNKYhrng0cJfz9H4+cEB6ohDIkXbGcik6DzECF5oK/vEW0OXqlpTl0D2Vrj23thFqJBUUSIhCKoeDabHDRO9dBHEmStZgK4LqTGVbO6pjgvUht5oi4nCZAGg8F43u7Wc7bmMlPUiNDb5hVIYVnD1+7JrO83cT5w1TeCdb3+MnXlHigZnOgko6RV8W0E9cViPvt0PA3//H/3MaWB/FTsRwd2ytJqNTfnux44qjTaaMZGr2u1E1XopUE35jTEzrHT38fz2oogFMHE2sSiTIYF3DZqTaaCUc424YIy4ylUvgd4Wk0TOlmGEYK36okZnbNtucj1KSrkwdWC1WhGalhgtg64t1+v1mqHvyTHT9wMiQhNaw+HJODxZhJgijbcCqKJ0s5mJcxV4xea5qsnyBs/Q96DYkA/nN4XJAmXAJHuuz0QENBUoZ7u5CEqxe4KdV176BhO1KVnTovn02cN2Jj9dAOqw9Lsyd2zHVv9mPdNYT9nyntMAf5Lsu97/Tv70H/s+HMK67ye+PfBKvv3RT3yeK9du8r0ffC+f/eKLfOn5y3zh2YsP+lZOtJ2I4A4bsafNF4wsl7oFF8o09CnjRTadrABapYGl4LulG1KUUUZ2+rsjFVFL9lh48jmTNY567BZQjP+uJZDVbtNqlT2jqjRNU4J4ESAbFwJFZMNgQUzkS1UZyvDffhhIMaKq1mnqvFEpUWyqk5ruek6lSKpjsPWlq3YTgK2Tte5AvPMM6zW5dMcep4a6oknvvSuCTFNWyoabPPYGUJ7TMTqrc9vv8xRem54j6ZQRtV1EP05dnRbQplO0xmc5LbBNXj+1k2OPP3aB73jfU3jvaAvDq+97zM/yPX17ue750H/4JF987iKfe+YlvvTCZfohPuhbOfF2QoJ7gUPy5MMKY6G0Bo9CHx+zslpcEzHMTkohdVqAhUzWzcCGDWRjfzklK35SC3J1rmo0HRRxYaNxUoPgZKSeNR4ZRFB/P1YtmREW0lEEq16DFZAioWlwPowZdNM0I8PDKJ1CUjvXer2ye9HMejmAE1IcwIUieTDgxNEWKWJXF56spGFAnSNFk0WIQ2HqlMEZKWfrTi2QRg3kqsYiQmOBR6TWsqlrbM6KSkKkUhCr4s6xjHvEwkvBM98t6lWD9Ka5bHOe6THH7W4GlMO5TVJwaifDrt24zdueeNPEtxXEsVqvyBrv6duzecef/KHv5f/6p/+OzxUYxjs3khpO7d52MoK7Uka3bb6OB/bKtMiZkfdaMWNV3UzyGQuspTAJiNMiBla0wkt+njfSiMawGLtO1Ryr8LxjDRglQHujl5DTJgA558vOwNkCE6NpwYdAjIpmZTab0c1MiwUgxrIgaBxhieVyaX+rYMoxJ9ONz0NReOzRGNGU0aiQM06UIGXAgURSeddijMxmM8vAh4hrwsju0ZSIqgRbLa1+4GSEi6w715PSJot2zpUB1KWTVRjvf6sYqhtW0TTg5oqSZaN4jn2xEzim3vuU8VIL2HUHNYWM7pX127OyP+iqFPSpnQhbrwee+fJFHrmwz9Ubd/iOb30azfqqvt14z0P7O/yFP/4H+NkPf5xbd454z9NP8OM/+SHiqdzvK9rJCO4wZs7TRE7L6wKjCqAPDtuRbz7YFZbJmhHNFQTeOn3FgGXkrNtJpQTqFFPBlTdZ58iUwVgyFYCpGuhhi2trei19srF7UYx+STYFyaZpSHFgtRKapqUrWhoxRVbrfqREVjgFMGmCbmYLUbQdRk6RHCNpSDhKrSErh7cOCN0MhycmE9QN3qHRGEXihNT3xqdP2aZH1b/jvQ3T1mnXpxu/V7HJUvWZ5HSsd0c3dZNpkZXy2hReUc2lCG57qeNyAfXeN8ffPW2pnvd4gK+/Uwc75EqTPQ3vJ8dUuXV7yc5ixqc/9wIf/M73orFQc1/Dt/d3dvljf/C3MxS9mZ/4uV9huVqTcuabUSv/teyEBHfDwzeZu71Wt27WvFRUIO1HlJrqiL+rbnRZpiP5xNnwZ4MfTMZXxbRbLCIZLi5ZydgfyJoLdh8JvsOHdsz6vdSOVLEZj24zPLvrOoNbiixwjD0ipjUz9APiVrTzhWWh0TLzCr/4gv/3/TB2r3pXGomA0DY0TcfRnTvkIRLXa9BMF1qGqDaAI0aG1ZImmAZ9Ggb6foDGk6K9B1Fsl0EOOPFINk66eM+Qa3HYAvgURvLejwNHUGPvVH0XhTKdfhKsZbu3YHxwCsF5amFbHGVhL0+s/l8q9l9om6VwLRh7CmUyOvD41Cb7/Q1j5mvrraf2+i2r8q9+7ldxIqz7yA/83u/i/Nk9mqbl6M5t8+3VxLeLYqT59hFNaFmEhmEY+G///H8M3vHMi5f53LMvcenqTZ557tJpoC92QoJ7wdEd1Ibu+nhESkeqZFwBX6yhodAkpxl8QRgQN/7bOW9NNs6XcXl+HBityYo3Fcexwc9ps1AgpGQyv8E3hNCSBiWVpcOXIsA02wbL8l3ODNEweNSZ4l3X0bQtKUWWKZampzIkIyfI0ARPRopUwKZY27YtLgRSzPSrFbnvaYMjrld4l3FZUfUE8TiX0RgZlktoGpw0li07hxZBMgJI9Ghr75HzTRGyLDBTCaSU4KipBOPybOy5lEHGajucray7vH9kW2qlIGwiUqAdM0eBdCpko7o13b7CVWORFRumbYE9b+3QataeUhrhNh2ThVM7KXbz4HD8/n/+mz/OY2+6wO/8rndzYa+hX97Dt3Xbt4kDabVkf9bgZi0f+JYn+c73PsXROvGRj32Wf/qvP/IA7+7k2AkK7lJ47RVqkfEnlamSUibFjPcbznOl4Uk9y4RZMaXnqWrRgK/ZpICzwhvi0Wwqk5a026zTrIrziurAOkZyMt2Z2smZo4X5qErwpvzYtK1dq1qz0HyxYGdnl9A0JVA6YowlCzZaZA26lQo5SuHCKIwVY2LWzWiahoPDJbnv0eBoG087c1ZTkBbNSuz7go+r4fBrG+RtHPyyqHkrPufCEkLqLNoxfBv27suwau/x2RNz2dWUbFpVrfGp4PD1PR8D/CTg36v7+PjrFUY5To3cPNOyucvWtDadsnX82PG8p5n7ibXrN27b1/UD/vQf+R5Wh0vScd92276dt3y7tyH1LrAza/jt7/8WPvTLn+TqN8Eg8teyr2ZYx9fM6kd7w2/f8KFrYK6Teip9sWbbI75bJvBU2uFxzHeqNmjoT4Vt/JjdB29fvnw5CeSYiMNAGgbiMNCv18R+RYo9aEbUBoAYfdL46euhJ2tivtjB+8Cq7zlarliuVty6dYvVamXMEW+QUQ3gG878sUYf7D7bWcfO3i5tN+PWwW36tanqkU2ioPHeZJDFtO6DgBeb/tT4YMNMUoJkHbDWD1YHleSRdWTvixvHTIVgjB4XmvH9tQBeMu1kUI0r2XwthG/x2Y8F8Lrb2YJS7Ea3MPp6/PHjtoXFtrn3tXlpeo5TO9n20svXOVzG4tu36Nf9ffl26wONbxBVNCUWXcPv+Z5vf8B3czLs5GTuW0G3BG6jY4zB3rlScBQ1VgpU0LZg5QWPrXiu1PPVQmmZ6uMdrgytICviwwT3ta64VGmRBSdyIjZYIAtkQWIEHagj3cRFXBxM9rcNOO9Zr9d4H8gaQYT5fEHXdgUvsux0uVwxn88BRt68K/i2L8OerXDraJqGhx5+hNXhETeuXONouWbezRAUHwzaGcf+ZQv8omrvSUqoWJdskmRaOSmRhoEchzLXtAzLLtRUVwTUKtRlAT5Y4SvXAnDhrHNMUiDX4msekRPZanLaDt4TRygFYHdXYB6DdjlX/fGUQ1+Ps+X/FI95I9mnvniJdz1+jqabFd+em283r+Xb9vkiK1ng8UfPMetaVutv7tGGJya4i26gleMMhymmnjXjtGbq5eeV3CjWxKOlMGgUOisgmjRvwIcW74Lh5mK0ypQScYiG4fswFjh9sKahyrvO2VrzTYTLVB+dlHFuTaDp2hFOMXqmlIwiM8TIerUGcXTzGbt7e6yHnpSVrAXXxtRz3AhLZNrWikfBlSlKbcOZ8w+x2N/n6ksvsr50jdXhmje/+REef+JNEECLbICI4HA2wLu8ZzWOxvWatQgSBzIwW8ytABrqU2CsJ9hAcLuytunohzU5Fw34FG1RdX4sLFfYKxTNGufcSHucMlym7Jhplj+16WJQh2VL3eGVGssoODY9R11QTjP3N4z9u4/8Bg//4d/Bo/tnufLiC6wuXWV1Z81jjz3K40/e7dv+FXz77Y9d4L/5iz/EP/k3/55PfeH5V/+jv4XtZAR3+9Qb7fEe3GXDgo0ZMy32mYKj0QwpRVA0l6afGjAcWRMuKyH48loqgUHAO2bNjNzmMhIvlcYkk+S1Jtk8NisZnl6zeVswQgi4xlgdMdrYMO+NjhdjBBUa723hcJ6um9lQalXaWbcl+wslA1aTEo5xGJ22au0szuzylrc+SVyvuf7yDQ6PIp///LPMF4FHHrqAir2P3jm04JOqdXw4qAjr1ZobN2/y0KOPsnvGsnVjhRpNsoJlufDafVn0MpnQNBu8nbJbynns3HUiYyAf72fE8s02XbF3d6gywmf2t+3QSeE6Vb3/kSm/cSXdJAcb3fpTe6PYv/yZX+E9b38TH3z3m7h+6TqHR5HPfe6ZjW+7jW/nV/Hthshv/8C7T4P7gzclk0YaZBmKhxg5EYcxU5wkvAhNcLRNQNSKKnG9JGeDFigiU0aB9Ih6nAuma67gUkKKBjtinaLrqkAZ+zFDz2kowaXIDog3Xfa6sBiygmgkDglPa3CPtyw8akLSQB8HchZ2dvbo5jOy2gi5NAxUTfoUSwEUNl2bqihWXK1yvGiR0w2O3bN7vO0972Jn5yIX/Qsc3IhcevEqF/bOgE9ISvjgGLwQcVx66TLXLl/lrW99gjPn9lgu18zPnmV2Zge8M0zLTemLsnmtZtdq0gs4Dz6ADkXLPm+xhmrdYNp0NMXFkQ3j5njGPc3qFYrcxGbcIIzjaQvuY08oj1z78izzRMLi1N4wtu4HPvXFi6z7yO/+trcR3BUObkYuvXBl27ebiW+/+DJXL1/lqaeeNN8+WvP8jUN+6kMff9C380DtfiYxPQH8GPAo9kn5UVX9GyJyHvh/gaewiTV/QlVviEWGvwH8YeAI+BFV/ehr/Z2KtdeJSqjNOq1rsiA0TSB4D2RWq5Xxv0VQTHVRECtSlkyvTkgy+drtkV1SC38lY86q4BxeHE4yqdAvFcsQBY9IAIQsVWBLjPY4FvJs8hFi+HlSxbnA3v6ZIh4mpJht/F02fr0rGi4VWui6rkgJZCQnAk251kyM/bgAeG+BeL47Z7G3w2p5hEPp1yvjvKeEEMAlXnzpCs8+8yVTtwyex/xjdLs77JzZNT59Y1OkprRCzdmuXzY6MNTtcAj4ZJLL5Iw4V3R0NuqZTIqaxxuPvPO2c1LdWgymOvP1ZxvfYGvBsIJwfX10os3zLd22J8G3T+0rs36IXL5+m9nOnHjGfFvk3r79wotXePaLzxbffom3uMeY7e2g3nPx8vUHfSsP1O4nc4/Af62qHxWRPeDXROTfAj8C/Kyq/nUR+SvAXwH+O+A/wuZLvhP4bmya/He/1h8Z5QZqQZTalARN42mCL5BJwcalUH0UqshJbcAxmqMj+FBmmlqBc8RtXeF6swkcMQ6os5Ry6AeqVnwsk5hC0Xhv2w6cYc2g9P3KdM1N4Zyjw8GG96ZImC04f/4hRBz9YJ2quYyISykxDD3OCW3n0NIBujw6GtvtY0zj4tH3K7yzBqrGeWgb/Nk9GufpQoOmyDxl0jAYwpLVGrW80DQznnjyrfjWs3t2l8XegsXeHrOdGS6UQFtkk2ECowgjF1+kSB1rLtm0jJREkDJeMI/3Jq42e20H7/KgtjT3a0De8OSLZo+U3ZdY7WIsk2brT9jAVRv1yHo+K6BnXqM7/Rvi26f2ldvhcs1PfvgTvO+pN3H+wrl7+3YQ2mbGE299itB6ds9tfLu99c1dTIX7G7N3EbhYvr8tIp8G3gL8EDZ/EuD/AH4e+wD8EPBjalHz34vIWRF5cznPa/ytiq9bVi0ITfA4sYnoFS0wanUp0JVM32h1pRgoYl2XYsJXuWDC21lkDSb276bpUJ8YYkSlDMfIxtpo2xlt6NCk9Ks1w7AqEsC5MHiEVZXxFeOmz2dzzjz0MPP5nH4wiCdlG9gxn89K8Mr0fU8cBpqmIRe5gxAC4hz9sDSoR6RIGkspzqbCblG6eYc8dI5HD9/E0aUrhZ2S0aSEYNe2u7fD4sycMA/sntvFNwHnO3BFB4cNk2XaNES2oF/lCpwIWn7B+0AOqfDet2UAiq+UGsZUPbJq6ehW9+rYhzCyaTze12dlgdrWmjzqAUmFYnJmbDkTa3GyS6+Q0Mnw7VP7ymy1HvjCc5e5cXDEn/mB70QODu72bSm+vX+3bz/30hWaJjB8E6tHfkWYu4g8BXwA+A/AoxOnvoRtbcE+HNMqxgvltdf8AIgTRE1/xLtAExyao2WVZRsvlKyv1OdM17t+iAVxHnzRaM9VsmDKzd5ABSlZZuwLmwMaMj0+ZHzweEmsVmviemB9tESTlmKuloCWTRQMa/Jpuw5xjmGING2HFzg6PCQma1oapRCWK6xRJ40Ux67rELHp79YIZXIGvtQKQmNNUkkjKUWcDzRgUJJrefjNb+bSwaHtboq6porJNIgXunlLs9PRzNrS0BQIwXZENYu+F/3Q1eJmGaRdA6YvAmQ5WSHaKGqb6VmuFohLkBcx2Yd6/qqPP30e0xF54/xUX7J5tuGZDWW2zn7dUGlr8bayfO7Hvt6+fWqvz64d3OHKnZ4n5ouSIB3z7TD17c6mjrmGD3zb0/zcL378NLjfj4nILvCPgf9KVW9Ng4CqqkxF1u/vfH8J+EsAO4tZPZFhsj4gGDZdPts1lo/Ydn3NOM8VR9/oyDixoB0Klly7Pium64LDlww5lazRiQl1ocry6A6r9RE5lQCWokkVCGQnRfvFmDJN0xLTQEwJTZnF7h5dN6NfrzfXKVakbULHrJuRM6NmddO0d+HMVSI4xgEBYjas2rRXHKkUE50IoevYPbNPt1igyyMUy3SzQEoDzWLBzpldur0FoW1AHCn2Yy3DFa17k7/fbi4SEdOWr5CJq9TGTfdsnxI6kVIYA+yxbH5U3Sx4W30W9e9M2TI1wNfCd6EtjcdOTRGQPBZRpzuFesRr+OLXzRtylNEAACAASURBVLdP7au30M1Qp2ge7vbt2YKdM3vM9hb0OXPj5pJz+47Vev1NHdjhPoO7iDSY8//fqvpPyssv1y2piLwZuFxefxF4YvLrj5fXtkxVfxT4UYALD+2P6KkFMBsS4R0j7dF47GUWKhYsNzS4yuiQMTh572w2ag0SrrSuF211o0ra1r1pmpJlwtCvLTN2jqbtGNaDMWcQ04YJgZgz3Ww26pushx4omu3Bxtstl0c0PpC9p+lmhGz606pS5AdqN6qADHYPpSGqFiO996WATJnnqog38bKUbKHxoexK2ob53hnurAe8KoIjYjuX2d4OzWKHpptZmHOO2bw1TD1nnAZqqSKXIG8yvkUTBilFX0EzpCLZ62BcNIciqYAqoTHa5FTpUtWak2qdxIZ0ZOsO9tuwjQ1Y2ebD14U7lwXWWEU1CJdjyuSsmsHXXcerxfavt29/pQvDqd1tL12+yR1R3vPo2W3f9jDb22WVhX/2Lz/Mcy9c5pnnLvH+b307s66la4JRjr9J7TXlBwpD4O8Cn1bV/3Xyo38B/Lny/Z8D/vnk9f9UzL4HOLhfTNKCgEnvGtWwMssZg51znlGWqig0Is5y0KLWqGoc9XHWaVF5rElgwoqCs9mMrrMReEMfWa9WpBhp28YCfmgIoTFIxHmGmBmyIs6zXvfj5KQQPIJl/lU7RnMmpcxstsCFBuebUYmy7/txrqr3NoR7GHpiNM12zbrJ+p2zmgFKVsPaYxqsAJwN97eZ3o5mNic7T04GcayTdaJ2OwvarqNtO5quo21bfNPiQmtZ/CjrYO+nFrhDlZHm6IMJr+WiYy8ihCbQNi1t1xLagPd2ret1TyyZ/BbjpUA0aGG6lOy+QjvTBqfpV3GAEcK5O4OvO7cC7RX10BzjqHfzoH371F6//dxHPs5nXrhMFHeXb4dZx4//5If52Q/9Op9/9iVSyvzax7/AL/7ypzhcrh/0pT9Qu5/M/XcCfxb4hIj8enntrwJ/HfhxEfmLwJeBP1F+9lMYVewLGF3sz7/WHxBKYM8DIgbFeFfJ0Pa9K1/AGIhK380WFotupF59wbhxG4xdBWazGaFpGIY4BiDvBcEC7Xq9pl+vLQDnTM6JpjEa5TCsyd5Z92XJSL0P+LYt2agVG0PTMJsvCG1rY/OGvgzIKLh0Mq59Wq8Rb4JeSYTgbRKTD55+vTbmjDjSEI3+WYqLMSVyTKz8mq6b04YW1zX4pmU4WuKDMVjmO7s2U9XZouObMAbcpgmWuVec24WS9kIuO4tMHUzuyrOwxaOyZWK2hbgp5/U+GP59jONenzOFxmhDs4WMmtpnKd5Om53G4K6Fvz5hQ2neHCdVuKzg9jblSceehFeRIfi6+/apffWmqjzz0hUu3ngLb541o28vdnaZdR3vefoJPvnZL3H0TR7Mj9v9sGU+TE2N7rbvv8fxCvzlr/RCrCinOBRf9WLIeO8Ks2KDtzutH9ntgc5SUnNXAvtUgtZgFssyY0oMRTe9XDNxSGUg9VBwbsFjk5SCd6xXK4bBZp26DGjDYr6DC46hj4g42vncgnEJ/FmF1XpFv16jMVoQbFvDzUvATpoIYF2fObNaL4lpoGmtwLo8WlrQdLY7UNRU8FRZrZbEtET3HNI5XLAh2H1K7Mxm+KYldM3I/x4nKZVipSlQ+nFIylCGFVe8v9Iasyq+FFOdd4ja88mqOOdtJ6SC9aGVjtaiQVOhmXH4dmH+mFaPdRyPP/OlhVw2xVWFsfBcn6UN3K47Cx29swb3WvRVeMWsvRz/DfHtU/vqbT1Ert9Z8VAjE99uAeV3ffe38xuffpZf/+QXH/Rlnig7IR2qIKVpyfBww9O9F0LwI86+aUffTEuajtqjvKolC5xm9BV+WS1XWMpf4B0bugoowTmS86gzjXcVGIaevl8Th2H8O13b4UPg6OgOOEfbWlD3zpqBYpkoo0BW66p1bct6uSKmFcPQUznioQy0zjoU+V1HCC3eNwWbz4RgOwVUGIYe7wNdNzPOfNmR9ENPRgldi3ohorTBs+p73HoNzrEIOwVjtyBYm7rqolhpqDXwVzpjKrUF5xwxmxJm27aAjRVMgw33FjVlTk3JJCDY3lXZIiLWQu5q/cTYLrEwb5ACq5Qh4jU/z6o20ERcKTyXc2eTm8i1KO7KOL7SXHW/bJlTO/nmgke94e1t41n2a2S9RnzipUvXHvTlnTg7IcFdETFRLleis/cQaqFNiwSAq4HYzKCBuz+8mjNZhJxhNp/jvePWrVuGE5fmpaZxuCCmkli28XEYLJsVx5AMAzdtF8sOu7ajbQPLO7c5vBPBebr5nOA9TdeAwBCtecI7Rz8M9P3K4BAXCqSRaZuO0DZGLcxK13TG2S8CZN43tF1HqAG23GsqeH9KGdVIKhDQarkixTJw2lvjD87Rzmes+4E2JVzfs/KO2XxuXPkUC/V0o9QIJphmT6R+GdafImQRW3icH98TRGi7mck49GubmGX4yzh/dSgLI1gQD2WerY4cfvuKZfqTk+1rmi4QGypltoY2VfDbTWmu1ClOVSF/i5nIxrdnG99uved3ffB9/ON/9YsP+gpPlJ2Q4G6sj+AduTAupIzuUc1okfdVVQsMbDjvdVMtpcNpZNV4z2Kxw7ofWK+O7G+UrM4y1ho0auEuj0XbNPTGFBFTlBQPi8Wcfr3i9q1bZZfBSOXLOTOURiQfgilArlekYcA54eyZM4hv6Fe9SQc0YdRs0axFVdGuOQQ7R9+vrdBZhLiybgLXECPDYLDSMESGvh+hjtVqTcyZRTDs2zcNy6MjNCvr3uQLQghIsN/1IZQoXouoalBNjGV+KphScHm/NYO6keaYyTjxLBZ7BB8Y1ktymcoUgreMvvD3c2Eu5CHhGo/TQKIUzsfCMdTRgsMw4JuwWeBSDfYFnisyBxv+/RaFcYuGeWpvbAvedmExJxZhbju5pmF5dDgK/nVtQ98Pp0t6sRMR3AWxuZopQuF+u7Itr/1KI65eeeCFbEEZW+1MxQvNhgvvzOcsD++gORHEuO9SxmrY0IrMOq23MPrgSxDFcPYYI75tEZTlnVukFPHO6IsOY+mEpqWZdTTtzEbNFVbIrOsIO7tjkNKUCI0DbChHjpkmNIb3J8PsvQb6oiMfvLfdipSxdGIZ7XJ1ZPfhHMOqp/GeRTej73vW/YqcB1ZxxVxmDHEwqEiMErZar8gpMd+Z03YdzgdWqoSmweXy3gbTkkeEpNkWG0qRdFR3LAuwmjyxTf0QmqbDi2cY1qRSt2iaZgy80RUMvmDpaGkKK5sA702XRzVBLLo+BarzhQqpZfFOmkkT7juTHgHULqnOWD39tL/x7ft/x7fyLY8/ws1rV5nL/C7fft/Tb+E973yCn/75j/JrnzjF3uGEBHdLCK0ByAL5pOV8LKSaPrgrfHQbjl0zuUlnpHfMZjNu375jlErAlUYm47nbaLnc9yQ2HOq0IV8Uhkyma1uGoWe9WpELT1vV4J4QAvtnzxHajtt3DtGjJW1rzUiLxcKyjCGWbNsy/BTjyHqxnQfWcOUD/3977xajS3bd9/3WvlTV131uHI40HN5Ey6Qk0oJIS6TshLEQBzZsS0YoJw+RjcQ2EOQCJIENJA9K8pIXPyRIFCBBEFiGBTiBFNrwJZIviSGbTghbEWVJoUSRBMmRRElkhhzO5Vy6+/uqal/ysNau+nqGl1EynO5zTi2gZ/p853R3VX271177v/7r/+/6DkQW5ydviarZ8YXY0XUDOWUqmVIqnQ9qyF0K4zgxTTP37z4wHRvP/mLPSYjUuRC7jtPTEyqoabY4YidU0WnRJn8QO1W3bNOkWZTV40w7BpM/TikpfFIqIUT79wrjxBh185kmlW8YBqN/lkvibfqsNbOXnM2T1V/C/hcYpmTETletWQrrmyatdyLrNHJKacPcH/IQEXZ95E1P3Obi7IJpTl91bd+5c4tSKu//nnfy+S88xwsvPbjqS7/yuB7JHUw4y+R6saElbBgJWY7uS9JfqrH1OB58R+wCh8NeIZOqVD6dvIEMSzO2VYvF8kzKK5sjxIh3wrlV63OaTfelEGPk5q1b9P3Aiy/dpZR73LhxiztP3EZEOBwOjLPJC2StLA8HxfJzSpRp1slb7zk9OUG8Z0qJ8/NzhmFQiAbh8LJkmOZZh4tEx/hLyewPe6IPnJ3tqQUe3Dvn7MEequOwn+hPe3JKuBBIaaYPA94JXdcvol611iUJNvikH/pl0rNGrYjD0fOvUmxzzNSSSOO86Nrr8NKMMzpoLtk0emRJvHKUgHPKlJwVdrGK3plsQSlFpb2cvl+Kudob5kQpsmJsn1VrYkn+jeK5xcMZJ0PHv/C+dxJd4Fbf8dJzL0D1urbdK9e2uMrf/8gv8dK9s6u+9GsR1yO52y+ya2YcTuly3tx9pFbDlBVakTbgJCxDSyJC33cKW4hWpM41J6C2Y9iJAIcY/NFcj5IZYO92O6Zx5HA4ADBNeyqwOznh9p1TUi5M88RhfMDNmze5desm+4sLvvLlZxFx3Lx1m5Iy4zQtmjYNm1dIQji5dYMudlxcXFDnya5JKLnQDzsmM7h2Zpo9jiOCDnBUYDg5pbgKznM4HDgcRvbnI3dfusdhv8eJcHZ2QXUwnJ7Y2D/mmuRwOdPHsCTBruuWZ1hK4bA/LHRIT4UQSCj33geTB+6AkhnnCWcTwLWqVHHOhXHaG6OGRYXzUPaMhwPee2KMy3tT0WbqIu+QTUSs6CSrqzaoZtOJbu2zg2HvOWcbDJNlWGrD3B/eeNOTt/m+93wbb/3WN/DcV+7y4gsvsF/W9rmu7RuX17Y4xwfe+y7+zv/+/FVf/rWI65HcWYdUQnBHHHUVrHKmYYKsfPbjfyOiCWocD+v0qmnFxNjRyvSu6xVWyZmctPGSZ63MpzRzujthmmZm+zvvPSenJ3S9duYP40g/DMzJEaNWl88/92Ucld5ro/Ol57+CWuntiCGSc2KaJsBxenrKjZs3uXfvHmfnZ4gl0KaZHkJcqlCpCsnM82TsGEwTHs7PzkCEeZw47A/LdG1NmRCUyz5NM900c3F+znB6AhjXvFQO5WBuS+C7aLIJgVIL0Zq8OjGrjdHCKiNQsmrJex+0OveRi4tzUp2IoUOcMBscddjvF/0Z7zwnJyfmLpUWM/DGrPGq/qW0R4FSTEZYoFTBu7gYo4uxJo4J6uK0Ga8uWXKk9b6xZh62eOqNt/g3/sQfIE0TUuGNN3ecPzh/5do+e+Xafu+738FnfuOLfOqzv33Fd3H1cS2Se8Olo2mXN+46teKCXwSkFGLBqvsmsOUJ1gRshVoIHu87vFPNmIph2/Y12WmDEMkGA0Tu3LzJeDiA9wxdt2LxZWScZ/qhZ7/fc//eXW7dukOaJ9J0oOaZ84sLcppBPCH23Lh1i9kSa98PdF1HCOqv+vyXvrxUyQ61C/POUYDD/gKjelsFrPBNmmfEeeO9JyrClJJVsQHvKie7geAc43gwPZzMNE2EMSyME8XWO0KMjONIBaI7ckmKQYW6VJqePCdqUnw/G2zlq0e1Ih0ZCK5jt4Pzi3vKwRdP13Wcn52r6JsIyaSO2zCVTrGq/EJlHWJqm0mtVQ28qUe68Ko71JQr24mtwWoVVEEyW+OXy5j9Fg9PvO+73op3QugH0jTb2hbG8cA0KsQ5TqOu7e7y2r596yZ/+od/gA//9Ef55Gce7wR/LZK7iBpyhOBMoVAZEa06b8qAPgSTCTD5XhsaKiWTK4jzdCEo9NowBzFdFHGIeDvCj0rtc57h5ISu65lTYth5qNpMjTGazymE2HP/7j12w8BTb36KB2f3GA/nHC7Ojd6nk5o3bpwCjvv37hK7nr7rySmz6wfyPHN2ca5G4IabZ5PK1SnQYiJHytgR0RMG2CRmSozjBCgrpYsDu90JVIVRgveMhwP9PjIdRtN3SRz2I4jy/Ydhtzzv4ALTOOLE0YXIPE10XnAxKvddCr7vNbmb3eA0Hog1UmsgGtWyJe4u7pjnSSlq3rE7GXSqNyWCD+RSmebJGE5lqdhB6ZTF7j0bdOVUO27B+UF7EZq0WTYKYKVQ0vSJMBMRa8x+zSHULa5bvPv3PM173vVt3Lp1C8rL13bHdBg5jCMp56+5tm/duMGP/Ks/wH//E3+X5164d8V3dHVxLZI7KByzWOqZAYb3zXNTlgq8KTZ6USErcQ7apKfx4kszbxbRqcbqcV4hkHlOpJxVPMuGX1JKSFU4pOt6ox8qDW9MKkL2xBufRErhuS89x/5wd6kOnQ/0Q0839Jw9OMOJ0HWdVhf7kTt37nC4uGCaVPcixG7h1/d9rxWxmWErl1eB5JyVbUMuykf3fhEy02Zlrz0H79md7sg50/sd0Xu8c8Q+kozVkqfM2d37uDuqHV/b5jXPzOOIR1lEcxq5efP2Mh0KICHgaqFU9bidDgdCiBBVumAuheoCXRiouVJKMiVOR9f3Nplq+jtOm921FlOsq2Rn/rhR4TSHkKlLzyWEuCh9tqjmlVuXjfJoJbVGbTv5yTfUxtviGsWXX7zPX/8HP88P/eHv4+1vfpKdt7UdXu3aznQxcjJ0/PF/+Xv557/yDG984iZvffOTfPh//ehV397rGtciuTfKo9KWLx+l2y/1OI0gntgpKyOEqJhszjhRca9WAYNx4xsv3r6dYt+i2K/9zs/zrNVnjHTDgBRN5lPSgaOTkxO8E9I0cu/eiwsTpIhytE9PT5imkfOzB/R9T5oTF4eRvh+4c/sW9+6+qBuS0fyaTkrf9+qzmrNZAfplRL+xVnItSBH6vmfYnUCDGrynH6JJ87pFgRHAdR0hKpc+zTMXFxfWJN3bycczNPkBS/CjDQIxFbrQE7rOrPR0ArYWdThChDll8qywzzAM1FJIRV2ydKNUnro2s009Mnhqhb4fcM7p4IkNRSlF3prkXq8vNg0J1s392KcWmhWgX7nsx/RKo88qA2eDZh6mePHeOXDO57/wHE89eYe+j8vavrufOL19g5LLpbW9v9hr8XC0tmPseO+738H3fc93UCl84UvPcbLrHytxsWuT3KE1v6D9UjeNk1IKPgQV/uoisYsI3oyk5zWxl+buY0yPEBCvGulqX6eVrxiVcJpVR73f7aBCTjO5aEXQ9z3eey72F+Q5s784U2ZGECiRYdcRgufs7B4CdF0kpZlS4dbNm3jvefH5ryhuDAvUUqswDD0XFxc6gem9Yc1eYY6kLk5TmgihI4RIP/RaweK1OeyElCZCiOSSFKOvZanKu5MdaZoNunKLrMI8Tdy7e5eUTjm9dZNiz1acQlfzrH2CLmWcF0L0iq63Kt55fK9QU86ZiwdnqrzpdWNa3JPw1kfxVJvmVWljVfKsA0zTYdXqF7VCVOVJG1xzYZFAaCyexqBpPFj93lq5H6fv9nXHa2uLhyt+9p/9Ks4Lb3nqCYJ3/PzHP0cMnifv3OAPfM87VcDu5Wv7pbuk+ZTT2y9f24E3PXmbf/2HPsj//Dc/ctW39rrFtUjusHa73VHjs42QhxDo+p7YddrkTAVxapIsuVhSVawXJ8aPj4s6o9IklbvdKuNSVYlRBQuqbQ6aEvqdVvAPHjyg6yL7UaV21WQbTk7UA/X8/BzvI9UGewThzu1bzHPi7osvGawki2wAIoToeHB2n2katULPM94FnCvLqSXnRD8MdN0OHyLTrD8/RFWc9MFRama/PzD0u2WDAPRndHHRfmnDRzFGUlHP1v3FnhCVoVOAbujIRTe0mouqWNZM7Dxup1rxjTLpnYegvzCH8cCUEq4WqhSg00q/JWrUGUk3VnSqF32fuhgXxoygss4pJypuaQArQmVCZ7ZOlgllw9ZLKWpszlqt6z8Qg4nKsRzRFg9R/MOP/goAf+gD38WTb7jFS/fO+Ln/+xm+/73vUpZXCJTzQuyiUpTb2u5eubaHfuC7v/PtvP0t38pvf/G5r/NTH524Fsm9Nt0UY0EcV/LNXzSEYA5AzqCCDLWQcgJxOI9NSVrV7w3CsOpVk7pW9LWqAmSl6hCNNTa9c3SdVtXj4cDJyQnjeNDk5hTvG/qe87P7qiUTbEinOnwXOT054fzBfcWlfTOW0IrXOUeM/aIy+fIJzPZ/5/TE0Q07uu6UlJR7H0JEvHLT56SiX0M/sNvtSCkxDAPjOGIaioRONXEapDEeDgx9R0W14M8ePMCHQOgi0zypj6x4nMeayolpVIu/GLqlSTnsdupRizCc7DgcDkuV1BhLq9G1GMSUORwO+sxLJeVZewzBk0tWy8GcVW4gZxK28TpZNudq7J/mQxtCwDmbdhV5JTRTq+rr2wTsFg9vfPJzX+D7vvvbuX3rlForX3rhLrdvnXB+cWDno/WqFNZLOXN239Z2v67tIo7T3cDbnn5yS+6vf+g0qncCFEqBGKMNuzhNAqCeqChVsNa8YuoNNxaDZUw9rjTnH9RQolW0SxXvHN6+1gXHfr9nmiZOTk51fB6FFHRq1bM/PwMqIahphcMz7AZcF7h39yVqmhcXKUA3HqcSA/lIYGzRMHeKxTdrO+fU4SiEYB6rjlyUv33an+g0Xt8b3LOeSkBZNznPdH2v95YTIUaKS8tkZwVl0qSJcZo4dTc47Pdqyl0qp6eniHfMc8Z7Tcb7/QUhdMssQncyqLywnajGw95gE5sqxQbL3GUv2CkpbznnxJwztYsL7CJOp5RLLqQKhYKPvbFndKDLeWUZ6b3q+68KBNa0yZlqiV5lLNyW2B+BePHuGf/Hxz7JH/rAd7E/TPyVD3+EJ+6ccvf+BW+4dcobb9/g7OLAv/ZH38+uC+va9sdrG05PI3/0B97H537ziwuL5gPv/b18+pkvcnZ+uOK7fO3j1djsvU1E/omIfEpEPikif8Fe/y9E5Isi8nH7+MGjr/lPReQZEfmMiPyxV3chxnywsf3gvCWFzJwyc6o40cSes3K9RTw+6pi7ygGDx9F5r4NK5sdZja2RclKdd5TBIVUFqbwThMJ+f8Y47hmGwVgXAR8H4nCiOPg8gXNI7Kk+IqHj9PYbEB84u/ciQtOB1yoYF9W3tDpCiAZP6D20qU1lkazHyGqV6JxmxBWczzgHu76j5JmcZmLQCc/YBaoUuiFSyPjOE7pAtL5ECFEhny7SnezoT3YMN045uXWDsDtVzH6eyFMi52oyuip5HEKHYudqlnFxcZ95HhkPF8yHgxqG10yw+YTGNVZrw0ouSXXwp1G17SvaQ+h6PZUMJ1TRRrJu1CuOnkuTapj0+tJMTQWK4Krgxev75pVhFJwniCP6QHCR2GYcXLSVdbVre4v//zHPmY/83CftdxpevHtOKZUX7p7x2d/6Es8+f5dP/Pr/w4MxcZ7N/ObS2s7knHj6qSf4T/79P8V3f+fbAXjXt7+Jf+fP/Cu8+11vucrb+6bEq6ncE/Af11p/WURuAr8kIj9rf/ff1lr/6+N/LCLvAX4E+H3Am4F/JCLfUWv9uk613hsjAm0YqlHGbBVjpeu6RSukVYOKepTFWEINM5zKzDqH1NWUImfFnTEmihc1n3aOxVZvnmaGYWfNURPRco6UE/M8WlNTkOIQKeyGgf3+nHE84J0np2rMD4WHuk6t/Pqh1wao96SSlsZjq95bM1H8qt8SY2QYeqVpCuSU2B9Ght1ONzKjB+ozUqYAAFWldkUiwQdKUX64t15GSFrNx67jwd2kxholE4xnP02elJPimCmtw0KlcHFxwcnJCfuLc05v3lykhr1NDOec1F8VAVfJaSKbmqQ6OyWVNfaeIs02UaDrFOc3bL/1WopNm5aMDiiVjJdAYw29DIlZ5AjWPzcu/Ncs31+Xtb3FNz9qhf/zY5/io7/wad75bU/xoT/8Xi04vsraPjnp+JN/5P285U1v5J3veJrbNwc++P7v4LO/8ewyrPcoxKux2XsWeNY+fyAinwa+3jb3IeDDtdYR+E0ReQb4fuD/+lpfoGwKASlGc6gr+8JrI7NVdqFNsTpnU451eU1x34w7aqCFEKgVlSEASip4CXin7ukX52dMhwM5Z7q+Y7fbUUolBjXjziUj2Xj2pdoDi4Tgubg4J5tEca2eGkAWuMXpkJLKouhpoRliH2mNt41HvGjii0rx7GKnmjtWSV+cny+SCLlkYtD7V5phtWtwplpZ6fsAVecHmjBYtO+dz8+Virk7Ue16gTRPJnM8Q05M83j0/OpiNn44HIhdRxhHYtcx26al7EUddspzUogmqdY8aD8jxk4lBqrBVkUtA8U7TmJHmmd1scqzukyVopAVmui981TysnlqYpe2Tqk1X+K8f6N4Pdb2Fq9v1Fp55re+zHMvnfP2p9/wNdf2nVsDP/RH3g81cRgPPP2tt3nizg2+8sL9q76F1yx+VxMeIvIO4PcDH7OX/kMR+VUR+QkReYO99hbgd46+7At8/V8YY6+0o/nq96nGzn5RA2yCUy3hiMgiL5tzXht7tSyYsNhQUZv0VLmCoDTGB/e5OFPOdd/3OvFprJjW0NNJVoeTQIw9sR/ohp2O/xt3XRk+ESdhGaxyXquErtfp10bmb0m9wTKgcgldF/FmKRiCskrmacI1FklVWpdyyBvunBQXrzbKX5Uzf9yk1X6BxwfBeQjBsdv1dDGy2+0YdieK0ZsGy5Tm5WdP48R+f2CaxsVPNc1anR/2++W1eZ4AdUaa54mUJ/aHPaVkUppVKXMcOez3zNPMNE0LhKOMGZ027oYdw+6UftjR9z29GZm7Zq5CteRt6qFHTenGlKk2cFVV2Bnl2Xxjusw3a21v8fpHrZWf/5Vf59O/8SV8iF91bc/TpOvS1vbJruM/+vN/jN/3HW+96st/zeJVJ3cRuQH8LeAv1lrvA/8j8HuB96HVz3/zu/nBIvLvisgvisgvXuwnVfM7SsA++EXC1Qev1aIl9WNopsyJNM2UOTGPkzYhvZphi2vTokkhg6Jj7zVn7t+9x7g/0HUd3TAwnCiursMQazQTbQAAGq1JREFUcWnUhuC1wdlF1WUJnTF0TA4hRELsF99TJ36hA0ZTXoSiU5nG8mhiWoumi3OWnFeuuNI7i1XNyYaBtMnqvVfmCE0ETT1Dm5mGiCw678eslXmeVW+9i3ivg1T9bkfXd3qtzpkMgipkqj7NuGjKizWec0rkpGJlJWf2F3tzXGKp4EUgpUwcBmKnNNRcdCOY08w0TzZNnJjHA8mu1zlHZxto7Hf0u1N2uxsMu1N8aHpBEUxsrZ3yFuaRmDl21Wf+CqzmdV7bv5uv2+K1i899/ln+zs/+Iv/slz/7qtf2MESe/pY7V33pr1m8KraMiER08f9krfVvA9Rav3z0938F+Hv2xy8Cbzv68rfaa5ei1vrjwI8DvOlbbldBG6LtyK01fKWLHbGLAIY/y5IA52mimI9mSwwuBELsLBmqQJZWhxBDpGTVTq+50Pc7XPC44AldDyh2fIyH2x3ixFGkGsTjEamULKs2TIYQrEB3ME0jfR/VDs851CPWU+uKYbcKO6UZ74MqTVKNUaMwUoNDVD7BLZRQXZDYxI5SOle4RxvIJZcFm26wVePcOyfErsOXQig6TaqboNIgFe92y4BV63vUwvJ95mnSDbcoN77v+oU/j3jE6wTp7rQnTzOgG1xKiVJVOrnve2W71OaJqpujNnSbDyuEIPiguj+4BsnUb8hhr0f//WrxzV7bIvKNjw1bvObR6NW//Inf5N3vfDPf+sYbCPVrru3Pf+EFnnryDr/z7KNjtP0Nk7toBvqrwKdrrT929PrThlkC/Cng1+zznwF+SkR+DG06vQv4hVdzMbUq06RUtdHrgk6BprlBIKoVk2eT5S1anXlzaGpVv+q0BC7Ozqho9e3EMaeJNM9QC12vlTjOEfueKmbUbBztJblbAhWniTwED8lRnaPmotrmCBJBioDT8fsQO2iwjjH1Si06xs9aXYJi7c47S1SyQi02VOWDoyJ2qgiUOalCoj405YanZMqLRv0UzJbO1Bxt0KlBWa3RmHLBZai1JyWHc5l5mmksk1ILOSstNWdrjBqDxlVvJ5TO4JCqks0+KBcdhZQE6E91aMk5R4dufrVU5pzpxFFywkugVllOKnrCaT61JhHsFYNvRUAxLnu1Z6EUUTFefnsDr35tb3E18dK9c/7yT/5jvue73sYHP/CdPHlHxf2O13apcO/+Of/0Fz7L+cWjI0/wair3DwL/FvAJEfm4vfafAX9aRN6HlkWfB/49gFrrJ0XkbwCfQtkI/8GrYRO0X97Ge+67HqmiptIIMQagMo4HpcUZoBScGStX8CFq1ecC82Eykw/dEOY2wQl0Q7/i3TGw2vrJ0jxsOL63QRk9GWglHWpQd6FgPp0Ahok775jmiS5G5vFwJHi1ukyt1bV5iYra2gVjtKiMQsVZksIkFNo16T0ZJbRoBVLNzSjNk07qlta4lQXCan6mTdoBoIueqWacC8QoiChtMc+JSiUsxhjK7Q8hUKqAzSM45wk+ULGTiHNU06RxPhBrR82zwlK2MQTv6fqBeZ6Z5pFUMtF7tVqk4H1nsEqTI8YaxdqMFbc2U5XmnqnFXLra861HjdYrXttbXG3Mc+aXPvF5fuuLL/BnPvQv8i1vuHFpbX/m15/l45/8bZ75rS8vU+qPQrwatsw/5auXP//g63zNXwL+0qu+CquUG51P1RtnalWaX+xVAXEcD5DVyKJBDRmbbrXkF5xjGkcV+HJC13eM44SPWmHvdv2SxBvmLQbHFPNOtXtYNgDQDadRLrM1bCV4XK34VplndQJyWeWKU0UZHrVV+CzJdaXpAVS8eYCWnMBkbUspNgSl4/oiKokbTHQMlJK5SCoUNZVOs2q906iGYBOycfFvRVZNfB8CoRZqbdK6kMU2GfsFWOmnQt91y4CSiEJGmPqiN5mFkpuSpQPfUXKiDYs2fHy3U9rpeNhTi84l6K9WNq2dsrxH7bmVWq0QaO5PoMr4msSbnMQCxXydAdXXZW1vcW3i+Rcf8FM//XP8mz/8QW7fOrETrorhPWqJHa7RhKr3fmk26oh8ITQ7NmA87JnzjCsmEetX/RllxGiTMKVJbekEQtdzdnFhuufQ9aqNklqDMiqLBliYH+vkqCU6YamkdcyoLDz0Bt8onGSQQTHHp5zVmLsWSi0szlK1Nft0IbVkR83kVJb7b9K/3utwDlWPkY36WKo+H9WAZ+05SBPTXU8Ja1NXMaZxPOi1WegmpoNHja0kzkFV8S5nyopNjM2HsDCRVJxMR79bM9WL2ubNaUKks2q/4lgNv1vCDqHDn3im6UAuiWSSwY2/fyz1KyI2/RtsQrXx8A07q2Ux9q6UV9NL3eIxi+dffMBf/qmPEPy6rg7j/MgldrgmyV2T2Wqu0Co2rbYL8zQtwzzemXlHhVqq6qn3DUOuXFycU0wD5XA4cHJ6SsmVYdiZAYc5GHk18FDoQvnxtbaEurJZsjVsjz8UE68LI8U5ByaEhcEXUy54FxSOQEfxTfmAXBPaZ6sLDKSxNlu99zhLpIojryP1hUI39FAqktv3UWmAdu2robi8gg+fUrK/U9pmLUUNTYw12E4Uq9yuNpSVPaTiZceJd5EQQPXli7FlcpqYrYo+ntBt99s2OB8CUXpcdkjSmQHq6sx0/NFAFu/bSasi4pdnoNfirEdSkLrpuW9xOfaH6aov4XWJa5HcG+XhuJnZm9HD/rAnJ6U3Bqej+lK1TShOKY8tAd2/f5+SVd1wniZObt5EnKM3bnyDXdZRd02u06Ta7eKU/96uY728eulrsmnSRDOWTipibj6vKnLlnKOG0M4Fuik4ocwFEeWkFxv6EakU8xBUqQRvMEhYKs9iXrIlZ+N9iw1QOVOErMsxUwzcV+qoUMpK91rvqVBL1oalbR4LC2kWemMR6AbbIBhranuPSLChoqRwSmnyu7I6IxV9ttg8QDNG8bYB1GpG1k4ZS+LEGE2NBFQXa8E2F+B8m2IttsnKcnppz6qaFnx7Dlv5vsXjGNcjubfjvVXtbYrzcNhTc1lMs733NNM0B6Znru5J9+/dXRJ7SjPDyamN4bdp1rIYMwPEvieXQkmJlNcR/oazr8l9rayXrzVZAk0+eamWleKneHYIyq3VzUpPHY394sSpEYcxVkqeDRu2BGrNXWfVKSJ4lB2Ti3HDS1Y9nlJfkbhLLXjCchI5Pg01KKVtqLXY11YhWwO6sYtKUTnVyy5I1quwk5KIEL1XyqJjwfPbxpVTYkKr6S52OmF7ZC6CTZ86r8atUio+rKeOrhRVnmycfcSGyxqH3xOjZ55Ha9qGSxv4Fls8rnE9kjuaa1R6t1JLYmp6Mc4bvTFQC8wGP/TWYDztB5UByJl+GEg5E3Y7hpMTrXIdpDzprKJZsw39DsxxSWEWdXZqDd3jDaYU8D6aY5LQ9wMhBtsYVK42hEC2it55TxUbl6cNJ7VEWCmWsJ14qDqEJOKMhhkUKhIQLzrOXwHnqa7YpKrD1cae0f6BMyldakH0QWrCrB5kvRd9zqu0Q/uziP4sj7J2Gtc/uLDI67bnUs1PtaKNX2qr6rXR7LxDiskhiMflRJlHavAUsepbzD/WKn1BZSFCUNZNzgplee+RENk5z+Ggqn31CJqrRVlOIMTYMU2Van2LbP0NdZB6XZfyFltci7geyd0GDkrR5FTRJLnqu4vyqIsxVLyzI3xgHPcc9nv6QRk1IUT6fjBLPfBF+dfFII/gI06qNV1tQtT7hc2iSdwvl9amYhvGHLuoGLgxdRbc2QaRKpCKuUeJp3FqGwcfKs5HHc2fZ5t01USvPYDGcDHao2vYvzZ1vflIguqyN363/k8TmrPX0pyIfbck8Ha/lwe0NJysDB7nBQqL+Ql2n22Stmn6tFibUe0Uoe9pCAGpKlWQ0kzzZm2GLAssxeq61Zqv2osoNiEc6cEYQrY5OX/ZpKPqHIDSQ6th7sJWwG/xuMb1SO6wYN9eLKlQ16N/S8wxUJ0Y3KKMiIuLc7q+o01ixk6t5tpQTypaMees1MHQdczTgZyyarSHxrwx9soRRXBlzByxRbynUK0xq4m5JRQnjlQrKWeTIjB+fMPeBaRUo0xmgxhseKlx/F2rRtef35JpG9YCHe0/pjM2ZclS8qUmbDZlx1WwzOCSbFCR6dEgxn/3thk5JRcuEJFBTN4HSmmDRWU55Qi6IVQT+1rej9gzTzPznAihMqdEeNlGk2ohhnUK+RiyaRDOSkuty/NoP2N9j1aHJj0ZrEbfW2zxuMU1oRJoYtfKzH5RrRoMPqisbPPpdF6bl85xdnZmYlut6lS4YhxHhTvsFz+nmZKSQhbWxItdIERviaIs9MRWob9c++WYQXMMbbQKU5ksSnjv+4GuiV55pzTBZRApLLK+4j3iFGtu9+0Ma4d12EmrfZt+pTJnxfaPsXD9ek1quTVrpTVrTWM9p+X7pTyT0mzc/WxJvyBV9Xd0M1qxfJF1k1ubq0fXWStS9IPaxCPU5q8bBqDBbpfpkAA1q5QBpVJNU749D8Cu0U4CBu28vDluTwBYTxkrTr/FFo9fXIvKveHtnTXvqmIYRFMD1CO9emuq3G1kv99bgjUvTufouqATrLXa14ri4kmHeyxDkWuGIpQjLnXOmWgNv2O2zFdLZLVqRUv7fEkm0MeBELW5Os+T7p6LcTeQ7Wt846frZG2LNZmqfs3lyvMyRVFcG6M2GEK5okuFXnJehrOmaVruK9uGNqeJruuYkxp156rJFfwinNYomymZAXd9JWW1lKLgkzVwSyngHV482ezP2jMOdq8tgXvnqUWVJrNolV6ano7zOKfPINeK2GRsqeBjpCTTwz9qFrfN14lTTfncBqO22OLximuR3DEoZk2k6/BQKSr+IK5C9QvUUWtRxUdL3N7pJlBLWcf4TXMl5wxS1Wd0mqgiRPMjbYYXbYKz0fSOGR0tYVwO0zERxXaDD4hXBUudtNRJyVK16m6G0blqIzLEjiLaIF059JqInOjUrA7oKGeclrxp0FFQUxM7sdTK0gyFNeG1jW/l0q9Ts8By77kU+q5bsX3RKvn4OczzfKk5C61qV8VHOXqtlkavFJVBts0wG5UTjhK8fc9GkxTvFvNsnZqtS5+jPddqMgSN/rkOazXGzhHEtgHvWzyGcT2SO1iilTayCYZH56xJwxmOHEJgHEe8Ew6HPQ7FiYPvmG3ytA3upGnSUfzmU2onhNjvAIzup/BB1/VLFdoSHlyuCoH1mC9ONU9EewO1Hnl2Wq+gVGewkZ4iUjK9lqDqkBI07zRYaPF7tSEczUmqYe4McipF6YBOhAKLkYlnxdSdC1a9GuPGojWLG9yj92j3VYri4fYznMFXzqlfqTJqFHJpjeFqU7SLvK4KvaBmh2LTaevPqYaXO3QeoDXSWz8jNwG0qqbmir+3HoxtdEdTvo1ZpJQdgaLX2DR1dF2ZNsQWWzxmcS2Su8BiKKHu9o7g4sKQiUY9HHYDh/2e4BV+cVT95Q9xsXgLPmjDsszkNBnOG9idnGiDzwcVFGvJJlf6YWdValkGfo71TI5jgWvaJiIFOWLXLJvLMmSzOi8pTKI4e87ZjAQKuSTKoupoJwVjhdTc6IbNoET9WDWfyVKhto0pl4yzayzZNgvTIj5OogqfqMqlGJRSlfep1XXR+x4PB/q+p02gLmwU+zOWTFNWPXfvo/HvBVxFSntW7fkpxu+kWQ3WBX5ahptyMcaLI5cZ8IQgKt8nqqtZsOGrRqsExHmcU61ubNPeGqpbPK5xPZK7QPSeaLxqTaitgl5x5ib9qzKydcGPoxPmWbWZ1aItkdJk/87Rh54QIpNh7MGr4NU0z6pBfonHfVnYa9GAOUoSx/j7MdbbIudM8J5aPIm8wALeB/AKY0g48lGdlbtdjxqEjRbakh6ik58KWzS/1PZ9/cIyKaZl45233oK+ycfX2RqS7V6CWfcd31+DYxqLaZEbyJnCMSOm2ulAVrqjq1QxgTfcsrG0oapSim1A66bUeh3RZBxSSngUeik5k20zWBq6GAXT9t6yJHiHq23id5W12GKLxy2uRXIH0YrWjurUqnrtteKc/rIPQ29JJjKNo2LxtdANO5PBVVZMLZmURnKeqc7Rd73ayJkWeYwR8Y5pmnHe0XU91LqMzB9LDRwrVba/O4ZnjpupinuXJYnpKUANQhr2XX1Z4RbDhadpopPeqvSkcNKRPHET6CrGk++63n4+6GAUy/fXKdsmWSA48SpaZvfdhNGGYVjv05qxQzcskNQxTbFJBjQ8fN1AjLl01K5cMHrJCG0IrGhlblLz4rRaz0mHwl5eWB8zXVJKampeL0/Y6s/SU8sx/p9LOaJ/6SnHb5X7Fo9pXI/k3nBTSxQq4lUIpryokgIJH7zJ+Sac5EVcqyZNWjklcp7JRRNUDJFh2DEMO03MTeLAmoZd11GNSy1Lc/K4aXmZIXPcXDx2Umo34X00qYFqVSz0/bB8bfv+DfsupVByxQU9jWSBPK5+qz4aV16bBXRdjzfOfClKbXy5Do7z66lDN8yV4rlcqehEZymZnJMJeukGACwbBejm004G7fvo929CZuum3Fxtaq1I1USOYeTeedsg1lOSSKHJBref2Z5p7DqKGZfr91fe+vFzb9DV8j7UZk2ywTFbbHEteO6aJNakqU07xWcBa0Zm8jwvIl1S1R2o5KLysvNEzjOlZGOaePp+R4yD4tVlrcZzTspuEVFcmssV+3ESbsnl5aqVy7UfwR2t6m4MnWHYLfz4YDK5wAJxtI0rdj3eGpmtSdqmOBtu7EMwX1in1EmRJVGO47icMlY1S92wun7VwenMh7YJszlnCpJOzOQahmFY3ouuW6mh6wlGE7t+dsSaufQssF6AR3CUXJc+Acsme1kJ9NJztaGrruuW9dAape3flaOkvvDZnRm3WN+mNVO3NL/F4xjXIrkDixb4QluzX+Kui2oWXbJVv8qYaAlnMifzpSFYwYnCF8FHSkVVJZ2/lAQUHjC9lqOBmJZEWmJ3RyPuxwbdTepXLJPVoiJiDU7o+n65Bu+DUhdNTjgYYyV4v4hpifE/G5vl5Vh+Mw1XKGQ1z8g5q6ql4fHqgOSNUaJm2H3fa8XrVk120OTtfMOy3XKi6Pt+oSm2ZN+s+dZDQnufDPd369CXIk6i0sqmlyPiFmaPsE6UHif49mzt7Vc4qQ0s1VVvv1FhXw6TLd+H9fm5LbVv8ZjG9YBlWKc+F1GrWm2as5LmETHjCLV2U42XeZxI037hhev4vkdCR4g7ChCcUFAmjROPeDHePJQ5EboOYy4qb9wSjhNH54M6npZkaoPZNpqyVOGtYs/ZlMvt+vuuV3peFaSA9ytEcewHCuupwTuxeae2kRRL0jrG78SDXV/wgZQnUpqgJBUbc6q8GHygZsO4RZRNlBS2in1ECPbvIZTAbMyebMNO0Qdm57Th3CsUVHK2wS1sqpbl2kttjknekrOz3F9BCogHUQnjkrPSV03wqxgspicDHQQTGrPHkX2GUsil4kR1h/TrCs647h7zqq3gZYWGbGltscVjGdemcj9mb7SKrjXzyqKDUi810qZp0tepZLQx6IKn63tjaGhlPc2TyvrG44R8uWIETEy4apVqlX2xalENPeYFO28iWinNCgmVhKkPWGK263WrJo33q8FFg2YUO294vCa8hj1rwlTs/JI5Rq0LTt1OF8eskxDi0k9oGjsKw9jbLWqSvQ5p6XAQQE5pHWzKWdUibRM6nko9xrQby0ebof6yLIF+dglGOn6f/dGA1QrP1GUCt51q2oYvjftvVX7j8Uv7efUy3t5OVlts8bjFNUnu9VLCUPMFZWNM47wkSmWg6C9zS4hNxaThqzpl6pbJxJxVfbHv+qPmY11gFYxO2WAC1bIRcsrGl18nORv+3pqh8zwtkNFCC2wfrEbUTSt+9YgNR9O4uqGoSYhWq+1ntH/T9HNajloS70JD1AGwks1qzwld19uQlkJZbXMR0Wla0J6Fj9agNePtUirjNC73Ok2TnaLWZ/oKOKRBU7VemjS+9A7XdYL0uGdRwfoA+dL3bajPpefgHBSlWLa/P/a9bUn/+Gce/9wttnicQl45Vn8FFyHyAPjMVV/HaxRPAs9f9UW8RvGo3Mu31Vq/5Sp+8La2r208KvfyNdf2NcHc+Uyt9f1XfRGvRYjIL273ssVRbGv7GsajdC9fK64JLLPFFltsscVrGVty32KLLbZ4BOO6JPcfv+oLeA1ju5ctjuNReobbvTxEcS0aqltsscUWW7y2cV0q9y222GKLLV7DuPLkLiJ/XEQ+IyLPiMiPXvX1fKMQkZ8QkedE5NeOXntCRH5WRD5n/3+DvS4i8t/Zvf2qiHzv1V35K0NE3iYi/0REPiUinxSRv2CvP5T3c91iW9tXE9u6trg0ePM6fwAe+HXg24EO+BXgPVd5Ta/imn8A+F7g145e+6+AH7XPfxT4L+3zHwT+N3T86A8CH7vq63/ZvTwNfK99fhP4LPCeh/V+rtPHtrav9D62dV3rlVfu3w88U2v9jVrrBHwY+NAVX9PXjVrrR4EXX/byh4C/Zp//NeCHj17/n6rGzwN3ROTp1+dKv3HUWp+ttf6yff4A+DTwFh7S+7lmsa3tK4ptXWtcdXJ/C/A7R3/+gr32sMVTtdZn7fMvAU/Z5w/N/YnIO4DfD3yMR+B+rkE8Ks/qoV4Lj/O6vurk/shF1XPeQ0VBEpEbwN8C/mKt9f7x3z2M97PFNycetrXwuK/rq07uXwTedvTnt9prD1t8uR3j7P/P2evX/v5EJKK/AD9Za/3b9vJDez/XKB6VZ/VQroVtXV99cv/nwLtE5PeISAf8CPAzV3xN/1/iZ4A/Z5//OeCnj17/s9aN/4PAvaNj4ZWHqATjXwU+XWv9saO/eijv55rFtravKLZ1bXHVHV20U/1ZlFnwn1/19byK6/1fgGeBGcXm/m3gjcA/Bj4H/CPgCfu3AvwPdm+fAN5/1df/snv5l9Cj6a8CH7ePH3xY7+e6fWxr+8ruY1vXtW4TqltsscUWj2JcNSyzxRZbbLHFNyG25L7FFlts8QjGlty32GKLLR7B2JL7FltsscUjGFty32KLLbZ4BGNL7ltsscUWj2BsyX2LLbbY4hGMLblvscUWWzyC8f8CIcF5FPKDtA0AAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - } - ], - "source": [ - "req = np.expand_dims(data[0], axis=0)\n", - "r = sc.explain(data=req, predictor=\"default\", transport=\"rest\", payload_type='ndarray', client_return_type=\"dict\")\n", - "\n", - "f, axarr = plt.subplots(1, 2)\n", - "\n", - "axarr[0].imshow(data[0])\n", - "axarr[1].imshow(r.response[\"data\"][\"anchor\"])\n", - "plt.show()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 49, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "seldondeployment.machinelearning.seldon.io \"image\" deleted\r\n" - ] - } - ], - "source": [ - "!kubectl delete -f resources/imagenet_explainer_grpc.yaml" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -883,14 +599,14 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Writing resources/cifar10_explainer.yaml\n" + "Overwriting resources/cifar10_explainer.yaml\n" ] } ], @@ -921,7 +637,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -938,15 +654,14 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Waiting for deployment \"cifar10-classifier-default-0-cifar10-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", - "deployment \"cifar10-classifier-default-0-cifar10-classifier\" successfully rolled out\n" + "deployment \"cifar10-classifier-default-0-cifar10-classifier\" successfully rolled out\r\n" ] } ], @@ -956,7 +671,7 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -973,9 +688,22 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 6, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING: Logging before flag parsing goes to stderr.\n", + "W1106 08:48:27.877613 139714443523840 deprecation.py:506] From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow_core/python/keras/initializers.py:143: calling RandomNormal.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Call initializer instance with the dtype argument instead of passing it to the constructor\n", + "W1106 08:48:27.892694 139714443523840 deprecation.py:506] From /home/clive/anaconda3/envs/seldon-core/lib/python3.6/site-packages/tensorflow_core/python/ops/resource_variable_ops.py:1630: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "If using Keras pass *_constraint arguments to layers.\n" + ] + }, { "name": "stdout", "output_type": "stream", @@ -1007,7 +735,7 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -1022,7 +750,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAUc0lEQVR4nO2dy24k6XGFI691r+KlWUP2ld09o1HLM5JhCdLCC8OAV34GP6E33lgLYwADhqWFZHhg2bJGM9090yS7SRabRRbrnne/QJwA2oDHAeF8ywz8VVmZeSqBOH9EBE3TCCHEH+H/9wkQQnQoTkKcQnES4hSKkxCnUJyEOCW2gn/3109gKneT43XJ6JF6PBwewjVpUsFYVG9grKnwuipqqcfrqIPPIwhg7PZmCmP/9uXvYGyxwucfxal6PLBvDaQyroeVma9FX1cFBf6yGv+3Rw2OhUEJY72u/rujIIJrqhL/5sFgAGPz+RzGsgz/7lj058p61ZU1PsfpfKU+dHxzEuIUipMQp1CchDiF4iTEKRQnIU6hOAlxipmvjwIcrmqcDm8n+vFnTz+Ca+7t9WFsdXsFY1eXFzC2rvRzrEKc1i63NYx99/oExua3CxirDXumLDP1eBTi8wgD/J/a7rRxrIVjeal7Y4FxHkWOnwGpsBXx+NE9GBsN9GduNVvCNfjqinS72DbLRnjlzfUMxvK1fq2OHhzBNXOwxoJvTkKcQnES4hSKkxCnUJyEOIXiJMQpFCchTrFLHwLgiYhIq6NXU4iIPH6gWybHY1wh0JRbGItibH2EI2wPLDN93Qa7A3J+Y1QqbLBdcvTRPoxFKahiEJEaVIp0ejjNH4Y41jLsktqojNis9fO4vV3DNb0e/q7NElsHQYltluVspR6fT/F9efxwDGM3U2zDdY3zP9jpwdgm1u2vg70uXGM8chC+OQlxCsVJiFMoTkKcQnES4hSKkxCnmNnaIsJ9W8YHeJPvTkvP5E6+wn12qhz32SkzPYMnIlKDze0iIlFL30zfbQ+N88BZwTTF2c5WF2f+Wm28+frps2fq8fUGb7y2cn+TCc5OWtnVvR0901jnemZSRAQ/HSJ5g6/V7D3+bbu7+rUaGb2AwhA7B0mCr/30Gm+mH++PYOz+R/rG/fUaP8PvLi9hDME3JyFOoTgJcQrFSYhTKE5CnEJxEuIUipMQp5hWyu0Gp4aTJbY3Tu70VHmwuIFrHh7ivjLZAqfe53Mca/d21ONlC29gj2q8YXsI+tuIiEQp/p/r9Y1Uf6xvOF/e4UKAbhf3W2on2AK4nlzDWAOsj8ePDuCa1dwYk5HjcxwNsKWTgks1v8Mb8M8v8HPVMWysssIFBN+e4Gs17Oq207rAz05mWEsIvjkJcQrFSYhTKE5CnEJxEuIUipMQp1CchDjFtFLOJrcwdvMe2xHtUq8UOTAqNw72rS4ruP6hzHGKOqvv1OPbW5wm7/Rx6n2vjS/XwkjL9/vY3ri8eK8eHw7xmjzHKXurSmc8xr120pZ+jXtdfO03a6siCPdNElC1JCKSN3oVTJzg85hcYSsl6+GqmtroxdQk+F5PgM2VJtY0b2toBFjzwSsIId8LFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKiRPcXn670G0KEZFsraeakwK34Z/d4c9LIr1yQ0SkrA0LpgBToxu8JgE2kIjItsEWwGSGraWT7AzGekO9cVUY4v/NjVEt1EJjxUUkMNL5gYDqmCWuBrmd4XsWxfharbe44qap9d92MMRN2YIAV87Ehm2TGTbcdIYbvVWiP1c7A3ztVyv8mxF8cxLiFIqTEKdQnIQ4heIkxCkUJyFOoTgJcYpppYz3cNOt0EjZV3d6Grpe49T7YolT1z2jmiWI8P9Lkuo/LzSslCbHsXaNLZ1UsE0xz3AavVzon5ltcTXF+ABbB1mG193c4OqN1UqfG/L8+RO4pgcaXYmITI1J1MMRrvyZg4nY312/hWvG4z0Y225xBU9eYislMp6rINDvWRzhZ8eq7kHwzUmIUyhOQpxCcRLiFIqTEKdQnIQ4xczWdo0Jyk8/+RjGxoOeevz09df4u9o42xmbfyE44xaAWJnhze1Fhjfn18Zm7jjG2et2irPNEdiYPb5njKcwMrkWvZ5+X0REdnb00RVbY5N6bPTZsTbZVxV+rlCWtNPDGV7rHFdbfD+jBN/Pfhdfq7DWM6/FBn9Xbmyyh9/zwSsIId8LFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKefEAt+/v9PCm58OHj9TjYYLths3dBYyt5lMYE6OlfhTpP68AvYVERCpjQvX+gyMYq0s8fuCtsQm8Br171itjanSDN+A/fPgAxkqjP1K/r9/PLNM3xIuIvHp5CmNxjB+t5QpPRUf9olCPIxGRIDA2qcOISFFgeyNOrE3s4PtyvLl9C/pqWfDNSYhTKE5CnEJxEuIUipMQp1CchDiF4iTEKbaV8vQhjL1+dwljk2t9Ina7vwvXtCOc5m9K3AdmPsdjEIJQ/3kNOC4isnes20AiIj/7q7/B51HhCoevXr2BsQZUaHSM/jzIIhIRuX//PoxtNzidf3r6Rj2eGtbST3/6FzCWxLiKZGX0krq80HsFlUtsv8yN0SCV4OcjMp4DEVxhksa6BTM+xNPIPzrUx25Y8M1JiFMoTkKcQnES4hSKkxCnUJyEOIXiJMQpppUSpUZDK2NEwqtXeiOvx2Nc1fH4nmGzHOFmS1WC161Weop9MMIVDk8//zmM7T96AWORUWnxzJqkDcomDvfxtWqMZmK1UYZRGKMJSmCzBCmutHjy5BjGWimu0mmMCpMcjJNY3GFL5M3JKxh7d/ISxqo7PJ5iu8Dfd3qpV0lNr/Caw4fY4kLwzUmIUyhOQpxCcRLiFIqTEKdQnIQ4heIkxCmmlVI1eGf+7gBXTRS7Q/X4CEyaFhFZbHCavzBmjfzgJ7/A6wq9Sdagj9P8u0d4knOW4XNczfRKHBGROsPNulp93SYq1tiaCVr4esRG47WojdcdHx+rx0PjnlmVM+v1GsasRmOjoT6zJU3wdw1GfRjbM6Zon/7+P2BsVuB7fXik21y3RlO2b88mMIbgm5MQp1CchDiF4iTEKRQnIU6hOAlxCsVJiFNMKyVscDq5j4sVZOeRPq8jCnGVyxe/+08YmxlT1v/2yScwdvRIt0VGQ5xez0BVhIjI22/+AGOTczw3JG0b1g2wN84vvoVrIsPCePriz/C6BJ9HK9bPI4iNWTQJfgiseS6LJa7eqGu9gic2mppJgO/ns49/iNcZ80tahnXzwwd6hcncqPr5h19+gc8DwDcnIU6hOAlxCsVJiFMoTkKcQnES4hQzW5uEOBvX6eDMn4i+7vwG92z55uoMxrYV7plzu8STl8ehPk6iSvAG8OkET9h++wb3o6lXMxhr7+7D2GqqX5P5FG+UTgu9sEBEJGjwtOZWy7hnIPNqTSyIYuP5aOMMatvYgF+XepY3CvAzEKV443uCmjSJiPwAb8APGvzbDo70ie/D8QFccz7DzweCb05CnEJxEuIUipMQp1CchDiF4iTEKRQnIU4xrZROhMcgBDFOy6PuMW9XxoZnYyf99hb3o7m81Cchi4g8f/5MD0T4Pyk3esckRk+lvRG2B8SwN1a3IMWe403Zu4NDGOsZFleY4NsdNHpRQhjhURKWlRKG+BpXFb4eFbhWndTqjYRtlibAG/C7Q2xx5QX+baev9CKHpy18Ho8f6/aLBd+chDiF4iTEKRQnIU6hOAlxCsVJiFMoTkKcYlelGLv9iwDv9r/L9EqRqwJbIoPxCMYCo3/M7O4axjZrULFS78E1u3s4lr74EYxFC3weN9f6hG0Rkc1WvyZxgCsm7u0b59jC1SAldkUkDIwgwij4sHoIlRX+bQKeq8CokJIAv2OC0DhJozop6WIb8QJMy94Wc7hmVmEbEcE3JyFOoTgJcQrFSYhTKE5CnEJxEuIUipMQp5hWihgTlJcbnBp+c/1OPb5osJUStfGpxD282/9q+R7GlqDtf4ALT+TgUG8KJiKyt4Mba92e4vEJaYVHNbRurtTjWWbYDUbXrdxohtYYXkpT6jZAleARGjGoZBGxp1dbHkyKmpAlRpULGOEgIlIV+DzyClcZCf5pEsb6Z4Yb/Hx3jXOE3/PBKwgh3wsUJyFOoTgJcQrFSYhTKE5CnEJxEuIU20oxmjS9m+oWgIjIm+tz9fg2xY2daqOKoREce3N2AmNfv/5aPf7k4TFc0zfskqSHq0H6Y5yWD9u4wiG60GNXZ6/hmo1hBS2Xhj1g2BthpU/0jvuGhWHcs6LA59Hp4MqZEDQNqwWfe2lMI1/cTGHsdoLn0TQLPNvk3kCv1hoYttNyu4ExBN+chDiF4iTEKRQnIU6hOAlxCsVJiFPMbO372S2MvZ7gMQhzkPnLjf+CpsKZP2sSchjjyda/+u2v1eNHR/fhmr/c2f1fnUfQwv2W4h38u0cgwxfExnelAxibz/H1qEucLR919Q3nkbFJfWVMFY9j/GhFhgsgYHxCZYzCWM5xRvbsj7+HseIWT1rfM0YryI6etQ8qvLm9NmIIvjkJcQrFSYhTKE5CnEJxEuIUipMQp1CchDjFtFK+u76AsestTqOXqZ5+b2psl4gRs1rqj+8fwNjlmb6x+V9/8y9wzfPjj2Hs8CNswVgjI9ZLbGEsNmBDdxdvwF8bfXGKLR4LIQ2+jnkOLJ0ltgDW6xWM7d/DU6NHIzx6A9U4RAEex7C8w2MQzs9w/6ZxB29UH3WxXYWoDLukZA8hQv50oDgJcQrFSYhTKE5CnEJxEuIUipMQp5hWyskMjzrYWitBxj6xssmB8YHGUOO4g/9fDh+N1eMn73B6/ZvXf4CxvV1sb8AxAiKSLYzRFS/1XkETMKZBRKSusDWTRtgeWCzwuIACVKwEgm/a3R2e2P3jn/wYxnbNyp+ufjzWj4uIRGKMDVni3j0tYxzDw0P92RERiYFtloL+RyIigWHbIPjmJMQpFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKuSlw5UkRGFOSQZv+KMD/BaHR9Mno+i+B8ZkpSl/H2Io4fYvHO/ziZz/H59EYzZ1ynLJfzPS2///9Cls6qzW2ZtoJvqW1UfmzBefYSrFFVBq/a/Zr3HTr6BBXEr349HP1eB3ge3Z4/wjGjj/9FMbenbyCsVvDZhn39XESeYHnZNTGSBEE35yEOIXiJMQpFCchTqE4CXEKxUmIUyhOQpxiWilZhCcGN1bDokAvS8HJcDtozfhIYjzTAtUItFu4QmA+x02rmgZXHZQ1bp6F23GJHBzpNsD+HbZ0yunW+ER8rYYDPFF6GOiVHaXhAEQVvvZXJ3hq9D9/8U/4Q3P9kTx6cgyXLBa4wdeiwff6psaVLl++wRVZnz3Vjwc5tlLSD+/vxTcnIV6hOAlxCsVJiFMoTkKcQnES4hQzW1uHOFVnTTxGE34Dq/dNiDOJrRRn1bodYx2YRN1pcM+ZPuhhIwJbI4mISLbFGdSqwvna8YG+CfzgRJ+eLCKSh7ggQYyeP90Ab2IXsHE/2+AePFZFwkEf9wk6/RZPRf/7yT+qx5OdR3DN5QSPoMhz7DhExgiNoMH38+RUz0R//vFjuObZeAfGEHxzEuIUipMQp1CchDiF4iTEKRQnIU6hOAlximmliLHRO6xwGr0b6JuNez1se/RSPOqg08L2htV7qN3WrYPuHrZSWkkfx2L8XcsMjzpojH40m6Wesu8Ym8qPunhUgNXHJsCuglQl6CFUYAOpMmybKOnBWOs+vp8nF3rhwcX7M7imMfpIRZExy8OqSKixNOYv9R5O76f6aA0RkfmP8FR0BN+chDiF4iTEKRQnIU6hOAlxCsVJiFMoTkKcYlopSYUth06EU/27Xd2O6Cb48+oMp+XzFc55h0Yavdrqds92gz2FeIB7zmRbbJeERrv9zRr3Jbo4P1ePFxk+x1aMz7HY4mvVNNgWaUX6vQkabH/VsWGzGL2HJu/x9ZgVeuVSlZozOWCoFmxjNcYIjTDEz3dT6s/3yTUekzH7zVcwBs/hg1cQQr4XKE5CnEJxEuIUipMQp1CchDiF4iTEKaaVshPdg7HBYIA/FNgbmTEJucyNEQO1OcgBgioStltsUxRGS/3V4g7Gul1caXHzHo8meP3qa/V42/i8ndE+jPVaxniKEDf46nb0+5nl2JrZNvh+zid4nMHbGbZS1jV4XxjFJY1RHWMNlLYqmmqjGZ2A0RsNqMYSEZlmnGxNyJ8MFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKGbZGMFZjx0FyUCSQ5zid3BjNlhpjanQQ4Bw7GNkiQYN/9s4IWxhVbVgHK1yR8O7yEsYmNzfq8Wd7eLZG0MK/OY5xrNfDTbe6Hf13Rzm2KaZXMxh7eYrnl9xtjJsN7J4mN94jRlVKaMSsZ8dwbqQJ9ecgMx7iRHB1D4JvTkKcQnES4hSKkxCnUJyEOIXiJMQpFCchTjGtlMDaSG80RwoiXfPtFt61nyQ41hinaRRNSNTolQXDIU6UHz/BFR9RiNPyb97qjbpERBqjGdqLz/5cPb4DrA0RkdiY/xEY52hVb6wWuvVRG/Nyria6DSQiMp3iG1NH+F6H4J5Jg98jgdHkTQRfj6IwZr0YzdCaQF9XxdhfjAv8mxF8cxLiFIqTEKdQnIQ4heIkxCkUJyFOMbO17dTY/hvgpWmqZ6aSGK8x9nJLZmRkF1u8GT0CGb57u3h69aCFNyjPLnBfnJu3FzB2OMCb2EejXfV4vcWZv6rCF6Qoce+bjdEfCWV510Y6/GKCN7eLOW0aZ6/DQD//JsTPjpWttTa+V7XRe8iI7Qz0ZyQwsuh319YYbR2+OQlxCsVJiFMoTkKcQnES4hSKkxCnUJyEOMW0UsZ7uIeQNbYgBBvmA7SpWUTKDR7HEBub4od9PGJgu12qxxfLW7gmNX5XleFYX7A90DHsmWqpjyZowIRnEZEksmwFfB4to4dQCeyIf//jf8E11zP9+oqIRC38XdZjFwGLrjZ7AVnvGFy9ERrvJqO9kHz2yRP1eL3EPZW+nOLCCATfnIQ4heIkxCkUJyFOoTgJcQrFSYhTKE5CnBI0zYdP3CWE/N/DNychTqE4CXEKxUmIUyhOQpxCcRLiFIqTEKf8D1IForCI9rrHAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAUc0lEQVR4nO2dy24k6XGFI691r+KlWUP2ld09o1HLM5JhCdLCC8OAV34GP6E33lgLYwADhqWFZHhg2bJGM9090yS7SRabRRbrnne/QJwA2oDHAeF8ywz8VVmZeSqBOH9EBE3TCCHEH+H/9wkQQnQoTkKcQnES4hSKkxCnUJyEOCW2gn/3109gKneT43XJ6JF6PBwewjVpUsFYVG9grKnwuipqqcfrqIPPIwhg7PZmCmP/9uXvYGyxwucfxal6PLBvDaQyroeVma9FX1cFBf6yGv+3Rw2OhUEJY72u/rujIIJrqhL/5sFgAGPz+RzGsgz/7lj058p61ZU1PsfpfKU+dHxzEuIUipMQp1CchDiF4iTEKRQnIU6hOAlxipmvjwIcrmqcDm8n+vFnTz+Ca+7t9WFsdXsFY1eXFzC2rvRzrEKc1i63NYx99/oExua3CxirDXumLDP1eBTi8wgD/J/a7rRxrIVjeal7Y4FxHkWOnwGpsBXx+NE9GBsN9GduNVvCNfjqinS72DbLRnjlzfUMxvK1fq2OHhzBNXOwxoJvTkKcQnES4hSKkxCnUJyEOIXiJMQpFCchTrFLHwLgiYhIq6NXU4iIPH6gWybHY1wh0JRbGItibH2EI2wPLDN93Qa7A3J+Y1QqbLBdcvTRPoxFKahiEJEaVIp0ejjNH4Y41jLsktqojNis9fO4vV3DNb0e/q7NElsHQYltluVspR6fT/F9efxwDGM3U2zDdY3zP9jpwdgm1u2vg70uXGM8chC+OQlxCsVJiFMoTkKcQnES4hSKkxCnmNnaIsJ9W8YHeJPvTkvP5E6+wn12qhz32SkzPYMnIlKDze0iIlFL30zfbQ+N88BZwTTF2c5WF2f+Wm28+frps2fq8fUGb7y2cn+TCc5OWtnVvR0901jnemZSRAQ/HSJ5g6/V7D3+bbu7+rUaGb2AwhA7B0mCr/30Gm+mH++PYOz+R/rG/fUaP8PvLi9hDME3JyFOoTgJcQrFSYhTKE5CnEJxEuIUipMQp5hWyu0Gp4aTJbY3Tu70VHmwuIFrHh7ivjLZAqfe53Mca/d21ONlC29gj2q8YXsI+tuIiEQp/p/r9Y1Uf6xvOF/e4UKAbhf3W2on2AK4nlzDWAOsj8ePDuCa1dwYk5HjcxwNsKWTgks1v8Mb8M8v8HPVMWysssIFBN+e4Gs17Oq207rAz05mWEsIvjkJcQrFSYhTKE5CnEJxEuIUipMQp1CchDjFtFLOJrcwdvMe2xHtUq8UOTAqNw72rS4ruP6hzHGKOqvv1OPbW5wm7/Rx6n2vjS/XwkjL9/vY3ri8eK8eHw7xmjzHKXurSmc8xr120pZ+jXtdfO03a6siCPdNElC1JCKSN3oVTJzg85hcYSsl6+GqmtroxdQk+F5PgM2VJtY0b2toBFjzwSsIId8LFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKiRPcXn670G0KEZFsraeakwK34Z/d4c9LIr1yQ0SkrA0LpgBToxu8JgE2kIjItsEWwGSGraWT7AzGekO9cVUY4v/NjVEt1EJjxUUkMNL5gYDqmCWuBrmd4XsWxfharbe44qap9d92MMRN2YIAV87Ehm2TGTbcdIYbvVWiP1c7A3ztVyv8mxF8cxLiFIqTEKdQnIQ4heIkxCkUJyFOoTgJcYpppYz3cNOt0EjZV3d6Grpe49T7YolT1z2jmiWI8P9Lkuo/LzSslCbHsXaNLZ1UsE0xz3AavVzon5ltcTXF+ABbB1mG193c4OqN1UqfG/L8+RO4pgcaXYmITI1J1MMRrvyZg4nY312/hWvG4z0Y225xBU9eYislMp6rINDvWRzhZ8eq7kHwzUmIUyhOQpxCcRLiFIqTEKdQnIQ4xczWdo0Jyk8/+RjGxoOeevz09df4u9o42xmbfyE44xaAWJnhze1Fhjfn18Zm7jjG2et2irPNEdiYPb5njKcwMrkWvZ5+X0REdnb00RVbY5N6bPTZsTbZVxV+rlCWtNPDGV7rHFdbfD+jBN/Pfhdfq7DWM6/FBn9Xbmyyh9/zwSsIId8LFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKefEAt+/v9PCm58OHj9TjYYLths3dBYyt5lMYE6OlfhTpP68AvYVERCpjQvX+gyMYq0s8fuCtsQm8Br171itjanSDN+A/fPgAxkqjP1K/r9/PLNM3xIuIvHp5CmNxjB+t5QpPRUf9olCPIxGRIDA2qcOISFFgeyNOrE3s4PtyvLl9C/pqWfDNSYhTKE5CnEJxEuIUipMQp1CchDiF4iTEKbaV8vQhjL1+dwljk2t9Ina7vwvXtCOc5m9K3AdmPsdjEIJQ/3kNOC4isnes20AiIj/7q7/B51HhCoevXr2BsQZUaHSM/jzIIhIRuX//PoxtNzidf3r6Rj2eGtbST3/6FzCWxLiKZGX0krq80HsFlUtsv8yN0SCV4OcjMp4DEVxhksa6BTM+xNPIPzrUx25Y8M1JiFMoTkKcQnES4hSKkxCnUJyEOIXiJMQpppUSpUZDK2NEwqtXeiOvx2Nc1fH4nmGzHOFmS1WC161Weop9MMIVDk8//zmM7T96AWORUWnxzJqkDcomDvfxtWqMZmK1UYZRGKMJSmCzBCmutHjy5BjGWimu0mmMCpMcjJNY3GFL5M3JKxh7d/ISxqo7PJ5iu8Dfd3qpV0lNr/Caw4fY4kLwzUmIUyhOQpxCcRLiFIqTEKdQnIQ4heIkxCmmlVI1eGf+7gBXTRS7Q/X4CEyaFhFZbHCavzBmjfzgJ7/A6wq9Sdagj9P8u0d4knOW4XNczfRKHBGROsPNulp93SYq1tiaCVr4esRG47WojdcdHx+rx0PjnlmVM+v1GsasRmOjoT6zJU3wdw1GfRjbM6Zon/7+P2BsVuB7fXik21y3RlO2b88mMIbgm5MQp1CchDiF4iTEKRQnIU6hOAlxCsVJiFNMKyVscDq5j4sVZOeRPq8jCnGVyxe/+08YmxlT1v/2yScwdvRIt0VGQ5xez0BVhIjI22/+AGOTczw3JG0b1g2wN84vvoVrIsPCePriz/C6BJ9HK9bPI4iNWTQJfgiseS6LJa7eqGu9gic2mppJgO/ns49/iNcZ80tahnXzwwd6hcncqPr5h19+gc8DwDcnIU6hOAlxCsVJiFMoTkKcQnES4hQzW5uEOBvX6eDMn4i+7vwG92z55uoMxrYV7plzu8STl8ehPk6iSvAG8OkET9h++wb3o6lXMxhr7+7D2GqqX5P5FG+UTgu9sEBEJGjwtOZWy7hnIPNqTSyIYuP5aOMMatvYgF+XepY3CvAzEKV443uCmjSJiPwAb8APGvzbDo70ie/D8QFccz7DzweCb05CnEJxEuIUipMQp1CchDiF4iTEKRQnIU4xrZROhMcgBDFOy6PuMW9XxoZnYyf99hb3o7m81Cchi4g8f/5MD0T4Pyk3esckRk+lvRG2B8SwN1a3IMWe403Zu4NDGOsZFleY4NsdNHpRQhjhURKWlRKG+BpXFb4eFbhWndTqjYRtlibAG/C7Q2xx5QX+baev9CKHpy18Ho8f6/aLBd+chDiF4iTEKRQnIU6hOAlxCsVJiFMoTkKcYlelGLv9iwDv9r/L9EqRqwJbIoPxCMYCo3/M7O4axjZrULFS78E1u3s4lr74EYxFC3weN9f6hG0Rkc1WvyZxgCsm7u0b59jC1SAldkUkDIwgwij4sHoIlRX+bQKeq8CokJIAv2OC0DhJozop6WIb8QJMy94Wc7hmVmEbEcE3JyFOoTgJcQrFSYhTKE5CnEJxEuIUipMQp5hWihgTlJcbnBp+c/1OPb5osJUStfGpxD282/9q+R7GlqDtf4ALT+TgUG8KJiKyt4Mba92e4vEJaYVHNbRurtTjWWbYDUbXrdxohtYYXkpT6jZAleARGjGoZBGxp1dbHkyKmpAlRpULGOEgIlIV+DzyClcZCf5pEsb6Z4Yb/Hx3jXOE3/PBKwgh3wsUJyFOoTgJcQrFSYhTKE5CnEJxEuIU20oxmjS9m+oWgIjIm+tz9fg2xY2daqOKoREce3N2AmNfv/5aPf7k4TFc0zfskqSHq0H6Y5yWD9u4wiG60GNXZ6/hmo1hBS2Xhj1g2BthpU/0jvuGhWHcs6LA59Hp4MqZEDQNqwWfe2lMI1/cTGHsdoLn0TQLPNvk3kCv1hoYttNyu4ExBN+chDiF4iTEKRQnIU6hOAlxCsVJiFPMbO372S2MvZ7gMQhzkPnLjf+CpsKZP2sSchjjyda/+u2v1eNHR/fhmr/c2f1fnUfQwv2W4h38u0cgwxfExnelAxibz/H1qEucLR919Q3nkbFJfWVMFY9j/GhFhgsgYHxCZYzCWM5xRvbsj7+HseIWT1rfM0YryI6etQ8qvLm9NmIIvjkJcQrFSYhTKE5CnEJxEuIUipMQp1CchDjFtFK+u76AsestTqOXqZ5+b2psl4gRs1rqj+8fwNjlmb6x+V9/8y9wzfPjj2Hs8CNswVgjI9ZLbGEsNmBDdxdvwF8bfXGKLR4LIQ2+jnkOLJ0ltgDW6xWM7d/DU6NHIzx6A9U4RAEex7C8w2MQzs9w/6ZxB29UH3WxXYWoDLukZA8hQv50oDgJcQrFSYhTKE5CnEJxEuIUipMQp5hWyskMjzrYWitBxj6xssmB8YHGUOO4g/9fDh+N1eMn73B6/ZvXf4CxvV1sb8AxAiKSLYzRFS/1XkETMKZBRKSusDWTRtgeWCzwuIACVKwEgm/a3R2e2P3jn/wYxnbNyp+ufjzWj4uIRGKMDVni3j0tYxzDw0P92RERiYFtloL+RyIigWHbIPjmJMQpFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKuSlw5UkRGFOSQZv+KMD/BaHR9Mno+i+B8ZkpSl/H2Io4fYvHO/ziZz/H59EYzZ1ynLJfzPS2///9Cls6qzW2ZtoJvqW1UfmzBefYSrFFVBq/a/Zr3HTr6BBXEr349HP1eB3ge3Z4/wjGjj/9FMbenbyCsVvDZhn39XESeYHnZNTGSBEE35yEOIXiJMQpFCchTqE4CXEKxUmIUyhOQpxiWilZhCcGN1bDokAvS8HJcDtozfhIYjzTAtUItFu4QmA+x02rmgZXHZQ1bp6F23GJHBzpNsD+HbZ0yunW+ER8rYYDPFF6GOiVHaXhAEQVvvZXJ3hq9D9/8U/4Q3P9kTx6cgyXLBa4wdeiwff6psaVLl++wRVZnz3Vjwc5tlLSD+/vxTcnIV6hOAlxCsVJiFMoTkKcQnES4hQzW1uHOFVnTTxGE34Dq/dNiDOJrRRn1bodYx2YRN1pcM+ZPuhhIwJbI4mISLbFGdSqwvna8YG+CfzgRJ+eLCKSh7ggQYyeP90Ab2IXsHE/2+AePFZFwkEf9wk6/RZPRf/7yT+qx5OdR3DN5QSPoMhz7DhExgiNoMH38+RUz0R//vFjuObZeAfGEHxzEuIUipMQp1CchDiF4iTEKRQnIU6hOAlximmliLHRO6xwGr0b6JuNez1se/RSPOqg08L2htV7qN3WrYPuHrZSWkkfx2L8XcsMjzpojH40m6Wesu8Ym8qPunhUgNXHJsCuglQl6CFUYAOpMmybKOnBWOs+vp8nF3rhwcX7M7imMfpIRZExy8OqSKixNOYv9R5O76f6aA0RkfmP8FR0BN+chDiF4iTEKRQnIU6hOAlxCsVJiFMoTkKcYlopSYUth06EU/27Xd2O6Cb48+oMp+XzFc55h0Yavdrqds92gz2FeIB7zmRbbJeERrv9zRr3Jbo4P1ePFxk+x1aMz7HY4mvVNNgWaUX6vQkabH/VsWGzGL2HJu/x9ZgVeuVSlZozOWCoFmxjNcYIjTDEz3dT6s/3yTUekzH7zVcwBs/hg1cQQr4XKE5CnEJxEuIUipMQp1CchDiF4iTEKaaVshPdg7HBYIA/FNgbmTEJucyNEQO1OcgBgioStltsUxRGS/3V4g7Gul1caXHzHo8meP3qa/V42/i8ndE+jPVaxniKEDf46nb0+5nl2JrZNvh+zid4nMHbGbZS1jV4XxjFJY1RHWMNlLYqmmqjGZ2A0RsNqMYSEZlmnGxNyJ8MFCchTqE4CXEKxUmIUyhOQpxCcRLiFNNKGbZGMFZjx0FyUCSQ5zid3BjNlhpjanQQ4Bw7GNkiQYN/9s4IWxhVbVgHK1yR8O7yEsYmNzfq8Wd7eLZG0MK/OY5xrNfDTbe6Hf13Rzm2KaZXMxh7eYrnl9xtjJsN7J4mN94jRlVKaMSsZ8dwbqQJ9ecgMx7iRHB1D4JvTkKcQnES4hSKkxCnUJyEOIXiJMQpFCchTjGtlMDaSG80RwoiXfPtFt61nyQ41hinaRRNSNTolQXDIU6UHz/BFR9RiNPyb97qjbpERBqjGdqLz/5cPb4DrA0RkdiY/xEY52hVb6wWuvVRG/Nyria6DSQiMp3iG1NH+F6H4J5Jg98jgdHkTQRfj6IwZr0YzdCaQF9XxdhfjAv8mxF8cxLiFIqTEKdQnIQ4heIkxCkUJyFOMbO17dTY/hvgpWmqZ6aSGK8x9nJLZmRkF1u8GT0CGb57u3h69aCFNyjPLnBfnJu3FzB2OMCb2EejXfV4vcWZv6rCF6Qoce+bjdEfCWV510Y6/GKCN7eLOW0aZ6/DQD//JsTPjpWttTa+V7XRe8iI7Qz0ZyQwsuh319YYbR2+OQlxCsVJiFMoTkKcQnES4hSKkxCnUJyEOMW0UsZ7uIeQNbYgBBvmA7SpWUTKDR7HEBub4od9PGJgu12qxxfLW7gmNX5XleFYX7A90DHsmWqpjyZowIRnEZEksmwFfB4to4dQCeyIf//jf8E11zP9+oqIRC38XdZjFwGLrjZ7AVnvGFy9ERrvJqO9kHz2yRP1eL3EPZW+nOLCCATfnIQ4heIkxCkUJyFOoTgJcQrFSYhTKE5CnBI0zYdP3CWE/N/DNychTqE4CXEKxUmIUyhOQpxCcRLiFIqTEKf8D1IForCI9rrHAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -1067,22 +795,22 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 56, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAVwUlEQVR4nO3de4xcZ3nH8e+zs7MXeze+rWMcO74QQkK45eKGABEKUFBASEmkKiKVaKpSTCsiNW2qKg1SCaJSQwWkVKBQ06QYRBMChBJVlCaNEGmKCLng2EkcO05ix3Zsr2+79tp7m5mnf8wYra3znF3vzmXt9/eRVp59nzlz3jmeZ8/MeeZ9X3N3ROTs19bqDohIcyjZRRKhZBdJhJJdJBFKdpFEKNlFEtE+k43N7Frg60AB+Fd3v2uS+6vOJ9Jg7m5Z7TbdOruZFYCtwEeAXcBTwE3u/mLONkp2kQaLkn0mb+OvBLa5+6vuPgY8AFw3g8cTkQaaSbIvA3ZO+H1XrU1EZqEZfWafCjNbC6xt9H5EJN9Mkn03cP6E35fX2k7i7uuAdaDP7CKtNJO38U8BF5rZajPrAD4JPFyfbolIvU37zO7uJTO7BfhvqqW3+9z9hbr1TETqatqlt2ntTG/jRRquEaU3ETmDKNlFEqFkF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEKNlFEqFkF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEzGgVVzPbDhwFykDJ3dfUo1MiUn/1WLL5g+5+oA6PIyINpLfxIomYabI78IiZPWNma+vRIRFpjJm+jb/a3Xeb2bnAo2b2krs/PvEOtT8C+kMg0mJ1W7LZzO4Ehtz9Kzn30ZLNIg1W9yWbzWyumfWeuA18FHh+uo8nIo01k7fxS4CfmNmJx/l3d/95XXolInVXt7fxU9qZ3saLNFzd38aLyJlFyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskQskukgglu0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskQskukgglu0gilOwiiVCyiyRi0uWfzOw+4BNAv7u/o9a2EPgBsArYDtzo7ocb182z08b/+qcwdqjcFcau+cSfnfa+7rn7jjBWaC+GsdWrV4ex40PHwtjWl7dktjujcT8sfs4dxblh7MjQwTD2+S/9SxhLzVTO7N8Brj2l7XbgMXe/EHis9ruIzGKTJnttvfVDpzRfB6yv3V4PXF/nfolInU33M/sSd99Tu72X6oquIjKLzWTJZgDc3fNWZzWztcDame5HRGZmumf2fWa2FKD2b390R3df5+5r3H3NNPclInUw3WR/GLi5dvtm4Kf16Y6INIq5h+/Aq3cwux+4BugD9gFfAP4DeBBYAeygWno79SJe1mPl7+ws9Mj6L4Wxd7//g2HswJEjYez1Xa+Fsc7gz/eKJavCbSrFjjiWczoYOx6X0ba+sDmz3briT44XvfXiMNbd1R3GnPhlNTI8nNk+cHAg3OalLZvC2J/e+g9hbLZwd8tqn/Qzu7vfFIQ+PKMeiUhT6Rt0IolQsoskQskukgglu0gilOwiiZi09FbXnZ0BpbdfPfitMDY2mj3Ka/68eLTWuasuCmPjmQWSqj17doexgwP7wtiCeedkts/rmBdu09Y9J4wVe3vDGOU4NHL0eGZ7oSseYTe3pyeMHR0aCmPj4+NhbNGCvsz20eF4m5GxuOz56pbfhrGXn3wijN32jZ+HsXqLSm86s4skQskukgglu0gilOwiiVCyiyRCyS6SiBlPXnG2GS3GZaiVb8kuoy1aEE+GODySPeoK4JUNT4exna9tDWOdc+IRYIvnZPd/+/YXw23ae+OS19uuuDLerjPuR1dwHNuKcb2x0BG/HPNKxIcH4xFslUp2fbCYM8kmbfH/59vfeXm83VB2uRHgm7fFJcxzV6/KbD+UM6rws3/zz3E/AjqziyRCyS6SCCW7SCKU7CKJULKLJEJX40/xwRv+KIxt3fB/me2lzvgK/t5dO8LYtpc2hrHKkf1hbM7iN4WxI3uzJ/o9tHdnuE3X2IIwZl6Kt+uKBwARXFkv5FwEby/GL8eeOfEV8jlBBQKgPJ59Fb/d4nn32rviQUMdHTmjl94dD65pq8TP7bxVyzLb1yzLbgf42wMHMtv/bX084EZndpFEKNlFEqFkF0mEkl0kEUp2kUQo2UUSMWnpzczuAz4B9Lv7O2ptdwKfAU7Uh+5w9581qpOzxfh4MDChUAi3GR0dCWOdHg90OHdRXGrC48nfjvZnl2QYjQdpLJ6/IoydMzce7NLWGdfR2irZZbm29rjveaW3tpxjXCrF5cFSUDqc2xkP/mmf0xnG3CphrGdhXBIdGYv7v3XTy5ntl3TH/bjwrdlluc6cOf6mcmb/DnBtRvvd7n5p7eesT3SRM92kye7ujwOTLtooIrPbTD6z32JmG83sPjOLv4IlIrPCdJP9HuAC4FJgD/DV6I5mttbMnjazeKYGEWm4aSW7u+9z97K7V4BvA+F0Ju6+zt3XuPua6XZSRGZuWsluZksn/HoD8Hx9uiMijTKV0tv9wDVAn5ntAr4AXGNmlwIObAc+28A+zhrHhgazA5Vzw20WL1kSxjqv+L0w1j6wJ4z17zkYxo4dP5rZXrR4RNbSN8X97+yOS4DjOcs/tbXlBCM5A8q8Epe8xkvxc8OyH7StkPPSt/gcaDklQDri0XcdvdnLcgHs2Jo9+vH4aHxd/EA5e9690lhczp002d39pozmeyfbTkRmF32DTiQRSnaRRCjZRRKhZBdJhJJdJBGacPI0DA4ezmy3eGAb5624IIwt6Yu/Zdy/NV6uqbMULw3VvW93ZvvwSE55KmcWyNFSPDGjj8flMB/PPlaljngkV9HjCSzHx3P6n1Oz6+wORu115IyiyynzlcfifoyWxsIY8dOmrZj9mIVjQ+E2vUEfC3HXdWYXSYWSXSQRSnaRRCjZRRKhZBdJhJJdJBEqvZ2Gj1z/J5ntr2/6VbjNvL6FYazznHhE3LzlcRmnMCceQdW+Izu2a1s8CvlYTulwcDCnnJRTDmsrDWe2F+fF55eyZ6/LBjA6FvejZ248Mq8QTGJZJu77+HB23wEG+veGsf074/X0fCCYCBRYOj97bbkFHXEpcvD4scz29pwRezqziyRCyS6SCCW7SCKU7CKJULKLJEJX4+vg1xt+HcY+1hfP7zZnTjxnWVt39hVagGJfPIhjUWf2iAsrxvuyrnhAzqHDwbx7QGU8XnZpUU/2AJT2nPPL4GC8r2IxHqxTyJsXzrKv8Jdylt4aPBxfcd/27JNhbLS/P4wt6Y4HFNGXXZWxUjyPXzmItQVz7oHO7CLJULKLJELJLpIIJbtIIpTsIolQsoskYirLP50PfBdYQnW5p3Xu/nUzWwj8AFhFdQmoG909e+Kxs9yNn/qrMPbCU+8MYyvOXx3GLGdeuKMjcclrYCgY4NEbD8gZGs2ZV+14vAwVlbjMMzKSPYijbTAuJx0dyl66CuBNS+NBQwsXLQpjBGNr2i1+6Q8ejF/Gr738chhbPjeeaG5h7/wwFonKawDj5WAOupwy5FTO7CXgNne/BLgK+JyZXQLcDjzm7hcCj9V+F5FZatJkd/c97v5s7fZRYDOwDLgOWF+723rg+kZ1UkRm7rQ+s5vZKuAy4ElgibufeI+3l+rbfBGZpab8dVkz6wF+DNzq7kdswtfy3N3Nsr+XaGZrgbUz7aiIzMyUzuxmVqSa6N9394dqzfvMbGktvhTI/GKwu69z9zXuvqYeHRaR6Zk02a16Cr8X2OzuX5sQehi4uXb7ZuCn9e+eiNTLVN7Gvx/4FLDJzDbU2u4A7gIeNLNPAzuAGxvTxTPbcy88FcaWnJszP11XPP/YyOGBMPbSxuy55nYGy0IBVMpxKa+zEPdjYCAulY0FI+KMeH2ig4cOhrH3ve+9YWxx7sjCnuz2YnY7QDvxCMHBwXhJpu5yPJLugpXLwlgxKLNaMH8egM3N/n9pzxkdOGmyu/sTxItpfXiy7UVkdtA36EQSoWQXSYSSXSQRSnaRRCjZRRKhCScb7A//+I4wtn/H74exNo9LVOXRuMQzcCB7maGnno9LgEeOxqW8OR1xKadciZdrOj6S3cfuruyJKAHGR+Ilng78LJ4EcuWKuKx1xWVXZbZXLC43rli1MoxdfNllYezVLZvCWH8pfm7L52WXAUdG43W5yjklzIjO7CKJULKLJELJLpIIJbtIIpTsIolQsoskQqW3FqpU4sM/njOZYzw9JJy3MrtstOTglnCbsX3Hcx4xnvRw4fy5ccyyR46Nx9U62kvxhI27tuwMYw89eH/8oKPZpcOVF10cbjJw+FAc87iP/ZXeMPb4S2+Esfe8LbvdgvIlQFdQeSuX4/8vndlFEqFkF0mEkl0kEUp2kUQo2UUSoavxLZT3l3b4eHyFvFSKr8cvX5Y9KOS8rfG0/qP7j+T0JL6622PxoBY8e7uRY8fibXIG1pw3b3EY2/riK2HsmzvXZ7Z39L0l3Ob1XfGSVyMjw2GsvT0eNGQe/39u2ZpdabjqnW8Nt3n78r7Mdrf4VaUzu0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJmLT0ZmbnA9+luiSzA+vc/etmdifwGWB/7a53uPvPGtXRM9U37vrLMNZVjP/WDuaUeDxnPrOjg9klnp5SR7jNyt54DrfRnPnuLO4ipfFgDrqxeIBPKWdetUJnPMike1Uc27Iju6y4441t4TZ55atCe07K5I1QqsRlucMbs+cAfGNf9lJeAIfXrM5sHxqOXxtTqbOXgNvc/Vkz6wWeMbNHa7G73f0rU3gMEWmxqaz1tgfYU7t91Mw2A/GpQERmpdP6zG5mq4DLgCdrTbeY2UYzu8/MFtS5byJSR1NOdjPrAX4M3OruR4B7gAuAS6me+b8abLfWzJ42s6fr0F8RmaYpJbuZFakm+vfd/SEAd9/n7mV3rwDfBq7M2tbd17n7GndfU69Oi8jpmzTZzcyAe4HN7v61Ce1LJ9ztBiC+dCgiLWfuOZOCAWZ2NfC/wCb4XW3kDuAmqm/hHdgOfLZ2MS/vsfJ3lpj9W58IY6VyvDzRpufjv6vbX9ue2X7gjf2Z7QBtOSsJDQ4eDWOeM09eR3v2XG02HtenKsX48QZzXjnbdsej9rbtzY4NUogf0OJ+5PGcJbsKxKXPwnh2rH0sXpZrUXf20lCbX9nCseHjmU9gKlfjnwCyNlZNXeQMom/QiSRCyS6SCCW7SCKU7CKJULKLJEITTrbQkYGDYaynNx7JtW/3rjD2wqbfZrbPyXm8vkXxZJS9XXHJqL0QTzjZO3d+ZvvwaFx6O16JR2wd2rU7jL16MC69DZWDElvOK99zRt+RUwIstMXnznIpLqVSzi71eVu81NTe4eyOjFc04aRI8pTsIolQsoskQskukgglu0gilOwiiVDprYVKOaWmw0cOh7FXd74exnb292e2v31J9tpgANYVvwyKOa+Qc86Jy3m9c3sy2wujcVlr7+4DYWzj1nhA5cGhnJkeC9lPwEdyznM5JbS2nBFxlnOw8hLNC9mTcw5X4ufV2TY36IRKbyLJU7KLJELJLpIIJbtIIpTsIolQsoskQqW3Bnvke3eFsfa2uIzz0ivbw5gX4pFoV7zn6sz2vqAUBlDMWb+sLaePlZzRYUcGsktlZY/3tXvnvjC2d29chiq3d4WxggejzTw+hpZ7DoyPx9hYOe5HzuScHsz4WSrG6+wVx7Kfc96MrjqziyRCyS6SCCW7SCKU7CKJULKLJGLSq/Fm1gU8DnTW7v8jd/+Cma0GHgAWAc8An3L3eGTHWeyH3/hiGJvfHQxYAA7seCOM7XtlRxhbMT8e1LJo0eLM9srx7OWCAEo586ON5SxDNTQaP6YFg0mGcuag27EzZ/Uwi5dram/PWVrJsvtfaSuG27Tl7CuvOlEq58xdR3ylvm9+9mvEcqokh97IPo6Ws5zbVM7so8CH3P3dVNd2u9bMrgK+DNzt7m8BDgOfnsJjiUiLTJrsXjVU+7VY+3HgQ8CPau3rgesb0kMRqYuprs9eMLMNQD/wKPAKMOD+u28s7AKWNaaLIlIPU0p2dy+7+6XAcuBK4OKp7sDM1prZ02b29DT7KCJ1cFpX4919APgF8F5gvpmduIKwHMicxd/d17n7GndfM6OeisiMTJrsZrbYzObXbncDHwE2U036P6jd7Wbgp43qpIjM3FQGwiwF1ptZgeofhwfd/T/N7EXgATP7e+C3wL0N7OesdnhwfxjrHI0HM5SH49g84nJST045rzSYvRTSeM4gjY72uAxlFvejK2dJqfG27PLVL5/9dbjNGwcGw1h79zlhrHrNOFshKLFV8uaSyym9kTP4p5Bz7rScTHvPuy7K3tNgPCffL/duz95P/LQmT3Z33whcltH+KtXP7yJyBtA36EQSoWQXSYSSXSQRSnaRRCjZRRJhnjNKpu47M9sPnBjO1QfEtYXmUT9Opn6c7Ezrx0p3zxz62NRkP2nHZk/Phm/VqR/qRyr90Nt4kUQo2UUS0cpkX9fCfU+kfpxM/TjZWdOPln1mF5Hm0tt4kUS0JNnN7Foz22Jm28zs9lb0odaP7Wa2ycw2NHNyDTO7z8z6zez5CW0LzexRM3u59u+CFvXjTjPbXTsmG8zs403ox/lm9gsze9HMXjCzv6i1N/WY5PSjqcfEzLrM7Ddm9lytH1+sta82sydrefMDyxuSmMXdm/oDFKhOa/VmoAN4Drik2f2o9WU70NeC/X4AuBx4fkLbPwK3127fDny5Rf24E/jrJh+PpcDltdu9wFbgkmYfk5x+NPWYUF1Qrqd2uwg8CVwFPAh8stb+LeDPT+dxW3FmvxLY5u6venXq6QeA61rQj5Zx98eBQ6c0X0d14k5o0gSeQT+azt33uPuztdtHqU6OsowmH5OcfjSVV9V9ktdWJPsyYOeE31s5WaUDj5jZM2a2tkV9OGGJu5+YOH0vsKSFfbnFzDbW3uY3/OPERGa2iur8CU/SwmNySj+gycekEZO8pn6B7mp3vxz4GPA5M/tAqzsE1b/s5K++20j3ABdQXSNgD/DVZu3YzHqAHwO3uvtJU+4085hk9KPpx8RnMMlrpBXJvhs4f8Lv4WSVjebuu2v/9gM/obUz7+wzs6UAtX/7W9EJd99Xe6FVgG/TpGNiZkWqCfZ9d3+o1tz0Y5LVj1Ydk9q+T3uS10grkv0p4MLalcUO4JPAw83uhJnNNbPeE7eBjwLP52/VUA9TnbgTWjiB54nkqrmBJhwTMzOqcxhudvevTQg19ZhE/Wj2MWnYJK/NusJ4ytXGj1O90vkK8PkW9eHNVCsBzwEvNLMfwP1U3w6OU/3s9Wmqa+Y9BrwM/A+wsEX9+B6wCdhINdmWNqEfV1N9i74R2FD7+Xizj0lOP5p6TIB3UZ3EdSPVPyx/N+E1+xtgG/BDoPN0HlffoBNJROoX6ESSoWQXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFE/D8jwvnE5fgcjQAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD5CAYAAADhukOtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAVwUlEQVR4nO3de4xcZ3nH8e+zs7MXeze+rWMcO74QQkK45eKGABEKUFBASEmkKiKVaKpSTCsiNW2qKg1SCaJSQwWkVKBQ06QYRBMChBJVlCaNEGmKCLng2EkcO05ix3Zsr2+79tp7m5mnf8wYra3znF3vzmXt9/eRVp59nzlz3jmeZ8/MeeZ9X3N3ROTs19bqDohIcyjZRRKhZBdJhJJdJBFKdpFEKNlFEtE+k43N7Frg60AB+Fd3v2uS+6vOJ9Jg7m5Z7TbdOruZFYCtwEeAXcBTwE3u/mLONkp2kQaLkn0mb+OvBLa5+6vuPgY8AFw3g8cTkQaaSbIvA3ZO+H1XrU1EZqEZfWafCjNbC6xt9H5EJN9Mkn03cP6E35fX2k7i7uuAdaDP7CKtNJO38U8BF5rZajPrAD4JPFyfbolIvU37zO7uJTO7BfhvqqW3+9z9hbr1TETqatqlt2ntTG/jRRquEaU3ETmDKNlFEqFkF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEKNlFEqFkF0mEkl0kEUp2kUQo2UUSoWQXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFEzGgVVzPbDhwFykDJ3dfUo1MiUn/1WLL5g+5+oA6PIyINpLfxIomYabI78IiZPWNma+vRIRFpjJm+jb/a3Xeb2bnAo2b2krs/PvEOtT8C+kMg0mJ1W7LZzO4Ehtz9Kzn30ZLNIg1W9yWbzWyumfWeuA18FHh+uo8nIo01k7fxS4CfmNmJx/l3d/95XXolInVXt7fxU9qZ3saLNFzd38aLyJlFyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskQskukgglu0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJULKLJELJLpIIJbtIIpTsIolQsoskQskukgglu0gilOwiiVCyiyRi0uWfzOw+4BNAv7u/o9a2EPgBsArYDtzo7ocb182z08b/+qcwdqjcFcau+cSfnfa+7rn7jjBWaC+GsdWrV4ex40PHwtjWl7dktjujcT8sfs4dxblh7MjQwTD2+S/9SxhLzVTO7N8Brj2l7XbgMXe/EHis9ruIzGKTJnttvfVDpzRfB6yv3V4PXF/nfolInU33M/sSd99Tu72X6oquIjKLzWTJZgDc3fNWZzWztcDame5HRGZmumf2fWa2FKD2b390R3df5+5r3H3NNPclInUw3WR/GLi5dvtm4Kf16Y6INIq5h+/Aq3cwux+4BugD9gFfAP4DeBBYAeygWno79SJe1mPl7+ws9Mj6L4Wxd7//g2HswJEjYez1Xa+Fsc7gz/eKJavCbSrFjjiWczoYOx6X0ba+sDmz3briT44XvfXiMNbd1R3GnPhlNTI8nNk+cHAg3OalLZvC2J/e+g9hbLZwd8tqn/Qzu7vfFIQ+PKMeiUhT6Rt0IolQsoskQskukgglu0gilOwiiZi09FbXnZ0BpbdfPfitMDY2mj3Ka/68eLTWuasuCmPjmQWSqj17doexgwP7wtiCeedkts/rmBdu09Y9J4wVe3vDGOU4NHL0eGZ7oSseYTe3pyeMHR0aCmPj4+NhbNGCvsz20eF4m5GxuOz56pbfhrGXn3wijN32jZ+HsXqLSm86s4skQskukgglu0gilOwiiVCyiyRCyS6SiBlPXnG2GS3GZaiVb8kuoy1aEE+GODySPeoK4JUNT4exna9tDWOdc+IRYIvnZPd/+/YXw23ae+OS19uuuDLerjPuR1dwHNuKcb2x0BG/HPNKxIcH4xFslUp2fbCYM8kmbfH/59vfeXm83VB2uRHgm7fFJcxzV6/KbD+UM6rws3/zz3E/AjqziyRCyS6SCCW7SCKU7CKJULKLJEJX40/xwRv+KIxt3fB/me2lzvgK/t5dO8LYtpc2hrHKkf1hbM7iN4WxI3uzJ/o9tHdnuE3X2IIwZl6Kt+uKBwARXFkv5FwEby/GL8eeOfEV8jlBBQKgPJ59Fb/d4nn32rviQUMdHTmjl94dD65pq8TP7bxVyzLb1yzLbgf42wMHMtv/bX084EZndpFEKNlFEqFkF0mEkl0kEUp2kUQo2UUSMWnpzczuAz4B9Lv7O2ptdwKfAU7Uh+5w9581qpOzxfh4MDChUAi3GR0dCWOdHg90OHdRXGrC48nfjvZnl2QYjQdpLJ6/IoydMzce7NLWGdfR2irZZbm29rjveaW3tpxjXCrF5cFSUDqc2xkP/mmf0xnG3CphrGdhXBIdGYv7v3XTy5ntl3TH/bjwrdlluc6cOf6mcmb/DnBtRvvd7n5p7eesT3SRM92kye7ujwOTLtooIrPbTD6z32JmG83sPjOLv4IlIrPCdJP9HuAC4FJgD/DV6I5mttbMnjazeKYGEWm4aSW7u+9z97K7V4BvA+F0Ju6+zt3XuPua6XZSRGZuWsluZksn/HoD8Hx9uiMijTKV0tv9wDVAn5ntAr4AXGNmlwIObAc+28A+zhrHhgazA5Vzw20WL1kSxjqv+L0w1j6wJ4z17zkYxo4dP5rZXrR4RNbSN8X97+yOS4DjOcs/tbXlBCM5A8q8Epe8xkvxc8OyH7StkPPSt/gcaDklQDri0XcdvdnLcgHs2Jo9+vH4aHxd/EA5e9690lhczp002d39pozmeyfbTkRmF32DTiQRSnaRRCjZRRKhZBdJhJJdJBGacPI0DA4ezmy3eGAb5624IIwt6Yu/Zdy/NV6uqbMULw3VvW93ZvvwSE55KmcWyNFSPDGjj8flMB/PPlaljngkV9HjCSzHx3P6n1Oz6+wORu115IyiyynzlcfifoyWxsIY8dOmrZj9mIVjQ+E2vUEfC3HXdWYXSYWSXSQRSnaRRCjZRRKhZBdJhJJdJBEqvZ2Gj1z/J5ntr2/6VbjNvL6FYazznHhE3LzlcRmnMCceQdW+Izu2a1s8CvlYTulwcDCnnJRTDmsrDWe2F+fF55eyZ6/LBjA6FvejZ248Mq8QTGJZJu77+HB23wEG+veGsf074/X0fCCYCBRYOj97bbkFHXEpcvD4scz29pwRezqziyRCyS6SCCW7SCKU7CKJULKLJEJX4+vg1xt+HcY+1hfP7zZnTjxnWVt39hVagGJfPIhjUWf2iAsrxvuyrnhAzqHDwbx7QGU8XnZpUU/2AJT2nPPL4GC8r2IxHqxTyJsXzrKv8Jdylt4aPBxfcd/27JNhbLS/P4wt6Y4HFNGXXZWxUjyPXzmItQVz7oHO7CLJULKLJELJLpIIJbtIIpTsIolQsoskYirLP50PfBdYQnW5p3Xu/nUzWwj8AFhFdQmoG909e+Kxs9yNn/qrMPbCU+8MYyvOXx3GLGdeuKMjcclrYCgY4NEbD8gZGs2ZV+14vAwVlbjMMzKSPYijbTAuJx0dyl66CuBNS+NBQwsXLQpjBGNr2i1+6Q8ejF/Gr738chhbPjeeaG5h7/wwFonKawDj5WAOupwy5FTO7CXgNne/BLgK+JyZXQLcDjzm7hcCj9V+F5FZatJkd/c97v5s7fZRYDOwDLgOWF+723rg+kZ1UkRm7rQ+s5vZKuAy4ElgibufeI+3l+rbfBGZpab8dVkz6wF+DNzq7kdswtfy3N3Nsr+XaGZrgbUz7aiIzMyUzuxmVqSa6N9394dqzfvMbGktvhTI/GKwu69z9zXuvqYeHRaR6Zk02a16Cr8X2OzuX5sQehi4uXb7ZuCn9e+eiNTLVN7Gvx/4FLDJzDbU2u4A7gIeNLNPAzuAGxvTxTPbcy88FcaWnJszP11XPP/YyOGBMPbSxuy55nYGy0IBVMpxKa+zEPdjYCAulY0FI+KMeH2ig4cOhrH3ve+9YWxx7sjCnuz2YnY7QDvxCMHBwXhJpu5yPJLugpXLwlgxKLNaMH8egM3N/n9pzxkdOGmyu/sTxItpfXiy7UVkdtA36EQSoWQXSYSSXSQRSnaRRCjZRRKhCScb7A//+I4wtn/H74exNo9LVOXRuMQzcCB7maGnno9LgEeOxqW8OR1xKadciZdrOj6S3cfuruyJKAHGR+Ilng78LJ4EcuWKuKx1xWVXZbZXLC43rli1MoxdfNllYezVLZvCWH8pfm7L52WXAUdG43W5yjklzIjO7CKJULKLJELJLpIIJbtIIpTsIolQsoskQqW3FqpU4sM/njOZYzw9JJy3MrtstOTglnCbsX3Hcx4xnvRw4fy5ccyyR46Nx9U62kvxhI27tuwMYw89eH/8oKPZpcOVF10cbjJw+FAc87iP/ZXeMPb4S2+Esfe8LbvdgvIlQFdQeSuX4/8vndlFEqFkF0mEkl0kEUp2kUQo2UUSoavxLZT3l3b4eHyFvFSKr8cvX5Y9KOS8rfG0/qP7j+T0JL6622PxoBY8e7uRY8fibXIG1pw3b3EY2/riK2HsmzvXZ7Z39L0l3Ob1XfGSVyMjw2GsvT0eNGQe/39u2ZpdabjqnW8Nt3n78r7Mdrf4VaUzu0gilOwiiVCyiyRCyS6SCCW7SCKU7CKJmLT0ZmbnA9+luiSzA+vc/etmdifwGWB/7a53uPvPGtXRM9U37vrLMNZVjP/WDuaUeDxnPrOjg9klnp5SR7jNyt54DrfRnPnuLO4ipfFgDrqxeIBPKWdetUJnPMike1Uc27Iju6y4441t4TZ55atCe07K5I1QqsRlucMbs+cAfGNf9lJeAIfXrM5sHxqOXxtTqbOXgNvc/Vkz6wWeMbNHa7G73f0rU3gMEWmxqaz1tgfYU7t91Mw2A/GpQERmpdP6zG5mq4DLgCdrTbeY2UYzu8/MFtS5byJSR1NOdjPrAX4M3OruR4B7gAuAS6me+b8abLfWzJ42s6fr0F8RmaYpJbuZFakm+vfd/SEAd9/n7mV3rwDfBq7M2tbd17n7GndfU69Oi8jpmzTZzcyAe4HN7v61Ce1LJ9ztBiC+dCgiLWfuOZOCAWZ2NfC/wCb4XW3kDuAmqm/hHdgOfLZ2MS/vsfJ3lpj9W58IY6VyvDzRpufjv6vbX9ue2X7gjf2Z7QBtOSsJDQ4eDWOeM09eR3v2XG02HtenKsX48QZzXjnbdsej9rbtzY4NUogf0OJ+5PGcJbsKxKXPwnh2rH0sXpZrUXf20lCbX9nCseHjmU9gKlfjnwCyNlZNXeQMom/QiSRCyS6SCCW7SCKU7CKJULKLJEITTrbQkYGDYaynNx7JtW/3rjD2wqbfZrbPyXm8vkXxZJS9XXHJqL0QTzjZO3d+ZvvwaFx6O16JR2wd2rU7jL16MC69DZWDElvOK99zRt+RUwIstMXnznIpLqVSzi71eVu81NTe4eyOjFc04aRI8pTsIolQsoskQskukgglu0gilOwiiVDprYVKOaWmw0cOh7FXd74exnb292e2v31J9tpgANYVvwyKOa+Qc86Jy3m9c3sy2wujcVlr7+4DYWzj1nhA5cGhnJkeC9lPwEdyznM5JbS2nBFxlnOw8hLNC9mTcw5X4ufV2TY36IRKbyLJU7KLJELJLpIIJbtIIpTsIolQsoskQqW3Bnvke3eFsfa2uIzz0ivbw5gX4pFoV7zn6sz2vqAUBlDMWb+sLaePlZzRYUcGsktlZY/3tXvnvjC2d29chiq3d4WxggejzTw+hpZ7DoyPx9hYOe5HzuScHsz4WSrG6+wVx7Kfc96MrjqziyRCyS6SCCW7SCKU7CKJULKLJGLSq/Fm1gU8DnTW7v8jd/+Cma0GHgAWAc8An3L3eGTHWeyH3/hiGJvfHQxYAA7seCOM7XtlRxhbMT8e1LJo0eLM9srx7OWCAEo586ON5SxDNTQaP6YFg0mGcuag27EzZ/Uwi5dram/PWVrJsvtfaSuG27Tl7CuvOlEq58xdR3ylvm9+9mvEcqokh97IPo6Ws5zbVM7so8CH3P3dVNd2u9bMrgK+DNzt7m8BDgOfnsJjiUiLTJrsXjVU+7VY+3HgQ8CPau3rgesb0kMRqYuprs9eMLMNQD/wKPAKMOD+u28s7AKWNaaLIlIPU0p2dy+7+6XAcuBK4OKp7sDM1prZ02b29DT7KCJ1cFpX4919APgF8F5gvpmduIKwHMicxd/d17n7GndfM6OeisiMTJrsZrbYzObXbncDHwE2U036P6jd7Wbgp43qpIjM3FQGwiwF1ptZgeofhwfd/T/N7EXgATP7e+C3wL0N7OesdnhwfxjrHI0HM5SH49g84nJST045rzSYvRTSeM4gjY72uAxlFvejK2dJqfG27PLVL5/9dbjNGwcGw1h79zlhrHrNOFshKLFV8uaSyym9kTP4p5Bz7rScTHvPuy7K3tNgPCffL/duz95P/LQmT3Z33whcltH+KtXP7yJyBtA36EQSoWQXSYSSXSQRSnaRRCjZRRJhnjNKpu47M9sPnBjO1QfEtYXmUT9Opn6c7Ezrx0p3zxz62NRkP2nHZk/Phm/VqR/qRyr90Nt4kUQo2UUS0cpkX9fCfU+kfpxM/TjZWdOPln1mF5Hm0tt4kUS0JNnN7Foz22Jm28zs9lb0odaP7Wa2ycw2NHNyDTO7z8z6zez5CW0LzexRM3u59u+CFvXjTjPbXTsmG8zs403ox/lm9gsze9HMXjCzv6i1N/WY5PSjqcfEzLrM7Ddm9lytH1+sta82sydrefMDyxuSmMXdm/oDFKhOa/VmoAN4Drik2f2o9WU70NeC/X4AuBx4fkLbPwK3127fDny5Rf24E/jrJh+PpcDltdu9wFbgkmYfk5x+NPWYUF1Qrqd2uwg8CVwFPAh8stb+LeDPT+dxW3FmvxLY5u6venXq6QeA61rQj5Zx98eBQ6c0X0d14k5o0gSeQT+azt33uPuztdtHqU6OsowmH5OcfjSVV9V9ktdWJPsyYOeE31s5WaUDj5jZM2a2tkV9OGGJu5+YOH0vsKSFfbnFzDbW3uY3/OPERGa2iur8CU/SwmNySj+gycekEZO8pn6B7mp3vxz4GPA5M/tAqzsE1b/s5K++20j3ABdQXSNgD/DVZu3YzHqAHwO3uvtJU+4085hk9KPpx8RnMMlrpBXJvhs4f8Lv4WSVjebuu2v/9gM/obUz7+wzs6UAtX/7W9EJd99Xe6FVgG/TpGNiZkWqCfZ9d3+o1tz0Y5LVj1Ydk9q+T3uS10grkv0p4MLalcUO4JPAw83uhJnNNbPeE7eBjwLP52/VUA9TnbgTWjiB54nkqrmBJhwTMzOqcxhudvevTQg19ZhE/Wj2MWnYJK/NusJ4ytXGj1O90vkK8PkW9eHNVCsBzwEvNLMfwP1U3w6OU/3s9Wmqa+Y9BrwM/A+wsEX9+B6wCdhINdmWNqEfV1N9i74R2FD7+Xizj0lOP5p6TIB3UZ3EdSPVPyx/N+E1+xtgG/BDoPN0HlffoBNJROoX6ESSoWQXSYSSXSQRSnaRRCjZRRKhZBdJhJJdJBFKdpFE/D8jwvnE5fgcjQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -1109,7 +837,7 @@ }, { "cell_type": "code", - "execution_count": 57, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -1137,7 +865,7 @@ }, { "cell_type": "code", - "execution_count": 58, + "execution_count": 49, "metadata": {}, "outputs": [ { @@ -1190,7 +918,7 @@ }, { "cell_type": "code", - "execution_count": 59, + "execution_count": 50, "metadata": {}, "outputs": [ { @@ -1221,7 +949,7 @@ }, { "cell_type": "code", - "execution_count": 60, + "execution_count": 51, "metadata": {}, "outputs": [], "source": [ @@ -1233,14 +961,14 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 52, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Writing resources/wine_explainer.yaml\n" + "Overwriting resources/wine_explainer.yaml\n" ] } ], @@ -1272,7 +1000,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": 53, "metadata": { "scrolled": true }, @@ -1291,14 +1019,15 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 54, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "deployment \"wine-default-0-classifier\" successfully rolled out\r\n" + "Waiting for deployment \"wine-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"wine-default-0-classifier\" successfully rolled out\n" ] } ], @@ -1308,7 +1037,7 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": 55, "metadata": { "scrolled": true }, @@ -1327,7 +1056,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 56, "metadata": { "scrolled": true }, @@ -1347,7 +1076,7 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 57, "metadata": { "scrolled": true }, @@ -1378,7 +1107,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 58, "metadata": {}, "outputs": [ { @@ -1404,7 +1133,7 @@ }, { "cell_type": "code", - "execution_count": 68, + "execution_count": 59, "metadata": { "scrolled": true }, @@ -1421,7 +1150,7 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": 60, "metadata": {}, "outputs": [], "source": [ @@ -1431,7 +1160,7 @@ }, { "cell_type": "code", - "execution_count": 70, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1441,14 +1170,14 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 62, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", - "
\n", + "
\n", "
\n", " Visualization omitted, Javascript library not loaded!
\n", " Have you run `initjs()` in this notebook? If this notebook was from another\n", @@ -1458,16 +1187,16 @@ "
\n", " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 71, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -1491,7 +1220,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": 63, "metadata": {}, "outputs": [ { @@ -1500,7 +1229,7 @@ "text": [ " % Total % Received % Xferd Average Speed Time Time Time Current\n", " Dload Upload Total Spent Left Speed\n", - "100 4131 100 3918 100 213 2518 136 0:00:01 0:00:01 --:--:-- 2517\n", + "100 4129 100 3916 100 213 1267 68 0:00:03 0:00:03 --:--:-- 1267\n", "\u001b[1;39m{\n", " \u001b[0m\u001b[34;1m\"meta\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", " \u001b[0m\u001b[34;1m\"name\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[0;32m\"KernelShap\"\u001b[0m\u001b[1;39m,\n", @@ -1528,53 +1257,53 @@ " \u001b[0m\u001b[34;1m\"shap_values\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", " \u001b[1;39m[\n", " \u001b[1;39m[\n", - " \u001b[0;39m-0.01848759963667984\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.005667299957192284\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.004803260335480397\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.07457241729089814\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.012711386372952105\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.14346065509049938\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.11803748686121629\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.02513961279583199\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.03775364762418021\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.12340641870395885\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.14947403055388797\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.24599462040520936\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.1294473840517647\u001b[0m\u001b[1;39m\n", + " \u001b[0;39m-0.018454421208892513\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.012763470836013313\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.001740270040221814\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.07633537428284093\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.006251732078452754\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.13734297799429607\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.1184209545879712\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.016528383221730947\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.035244307767622385\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.1230174198090298\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.14524487323369617\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.2507145522127333\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.1309476564266916\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", " \u001b[1;39m[\n", " \u001b[1;39m[\n", - " \u001b[0;39m0.011603013981827892\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.03815998485570904\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.04416372625394949\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.005147507199704582\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.04508317529365635\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.012338039536190781\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.05028475252605069\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.012131704105884156\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.04323714654943561\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.15578058674560102\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.15263596597664542\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.10139543659293687\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.06589311375577023\u001b[0m\u001b[1;39m\n", + " \u001b[0;39m0.015452322719423872\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.03832005916153608\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.04544081251256327\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-7.582651671816931e-05\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.04651784436924722\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.01980318229602418\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.05109329519459854\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.0071260827408027305\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.03975300296252354\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.16144118472393876\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.15110853673546232\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.1006346397643918\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.06837621797012206\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", " \u001b[1;39m[\n", " \u001b[1;39m[\n", - " \u001b[0;39m0.010036212941881395\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.028172552599049383\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.043733781243482595\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.06884057695884205\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.02586736063120587\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.13618579047327894\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.1683622966340027\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m-0.0347404026399003\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.07959663165005271\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.28251088076239483\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.3005435627422499\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.34968279298664484\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.07031415188992796\u001b[0m\u001b[1;39m\n", + " \u001b[0;39m0.005403884749745513\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.020371174031396766\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.04217363001738739\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.07712885770720579\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.03400723775835046\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1216306489876593\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.16988456143104136\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m-0.02030618995668898\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.07351750311197458\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.2879617844043466\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.29473542234118644\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.3536965903131064\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.06890108397640105\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", @@ -1630,19 +1359,19 @@ " \u001b[0m\u001b[34;1m\"importances\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", " \u001b[0m\u001b[34;1m\"0\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", " \u001b[0m\u001b[34;1m\"ranked_effect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", - " \u001b[0;39m0.24599462040520936\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.14947403055388797\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.14346065509049938\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.1294473840517647\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.12340641870395885\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.11803748686121629\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.07457241729089814\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.03775364762418021\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.02513961279583199\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.01848759963667984\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.012711386372952105\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.005667299957192284\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.004803260335480397\u001b[0m\u001b[1;39m\n", + " \u001b[0;39m0.2507145522127333\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.14524487323369617\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.13734297799429607\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1309476564266916\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1230174198090298\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1184209545879712\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.07633537428284093\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.035244307767622385\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.018454421208892513\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.016528383221730947\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.012763470836013313\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.006251732078452754\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.001740270040221814\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", " \u001b[0m\u001b[34;1m\"names\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", " \u001b[0;32m\"od280/od315_of_diluted_wines\"\u001b[0m\u001b[1;39m,\n", @@ -1653,28 +1382,28 @@ " \u001b[0;32m\"flavanoids\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"alcalinity_of_ash\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"proanthocyanins\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"alcohol\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"magnesium\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"malic_acid\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"magnesium\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"ash\"\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", " \u001b[0m\u001b[34;1m\"1\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", " \u001b[0m\u001b[34;1m\"ranked_effect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", - " \u001b[0;39m0.15578058674560102\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.15263596597664542\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.10139543659293687\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.06589311375577023\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.05028475252605069\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.04508317529365635\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.04416372625394949\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.04323714654943561\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.03815998485570904\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.012338039536190781\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.012131704105884156\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.011603013981827892\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.005147507199704582\u001b[0m\u001b[1;39m\n", + " \u001b[0;39m0.16144118472393876\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.15110853673546232\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1006346397643918\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.06837621797012206\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.05109329519459854\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.04651784436924722\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.04544081251256327\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.03975300296252354\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.03832005916153608\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.01980318229602418\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.015452322719423872\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.0071260827408027305\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m7.582651671816931e-05\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", " \u001b[0m\u001b[34;1m\"names\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", " \u001b[0;32m\"color_intensity\"\u001b[0m\u001b[1;39m,\n", @@ -1687,26 +1416,26 @@ " \u001b[0;32m\"proanthocyanins\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"malic_acid\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"total_phenols\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"alcohol\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"alcalinity_of_ash\"\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", " \u001b[0m\u001b[34;1m\"2\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", " \u001b[0m\u001b[34;1m\"ranked_effect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", - " \u001b[0;39m0.34968279298664484\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.3005435627422499\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.28251088076239483\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.1683622966340027\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.13618579047327894\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.07959663165005271\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.07031415188992796\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.06884057695884205\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.043733781243482595\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.0347404026399003\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.028172552599049383\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.02586736063120587\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.010036212941881395\u001b[0m\u001b[1;39m\n", + " \u001b[0;39m0.3536965903131064\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.29473542234118644\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.2879617844043466\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.16988456143104136\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1216306489876593\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.07712885770720579\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.07351750311197458\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.06890108397640105\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.04217363001738739\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.03400723775835046\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.020371174031396766\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.02030618995668898\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.005403884749745513\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", " \u001b[0m\u001b[34;1m\"names\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", " \u001b[0;32m\"od280/od315_of_diluted_wines\"\u001b[0m\u001b[1;39m,\n", @@ -1714,31 +1443,31 @@ " \u001b[0;32m\"color_intensity\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"flavanoids\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"total_phenols\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"alcalinity_of_ash\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"proanthocyanins\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"proline\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"alcalinity_of_ash\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"ash\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"malic_acid\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"magnesium\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"malic_acid\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"alcohol\"\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m}\u001b[0m\u001b[1;39m,\n", " \u001b[0m\u001b[34;1m\"aggregated\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m{\n", " \u001b[0m\u001b[34;1m\"ranked_effect\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", - " \u001b[0;39m0.6970728499847911\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.6026535592727833\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.5616978862119547\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.3366845360212697\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.2919844850999691\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.2656546496974629\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.16058742582366853\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.14856050144944477\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.09270076783291248\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.08366192229781433\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.07201171954161645\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.0719998374119507\u001b[0m\u001b[1;39m,\n", - " \u001b[0;39m0.04012682656038913\u001b[0m\u001b[1;39m\n", + " \u001b[0;39m0.7050457822902315\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.5910888323103449\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.5724203889373152\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.3393988112136111\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.27877680927797954\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.2682249583732147\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1535400585067649\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.1485148138421205\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.08935471257017247\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.08677681420605043\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.07145470402894616\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.04396065591922266\u001b[0m\u001b[1;39m,\n", + " \u001b[0;39m0.0393106286780619\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m,\n", " \u001b[0m\u001b[34;1m\"names\"\u001b[0m\u001b[1;39m: \u001b[0m\u001b[1;39m[\n", " \u001b[0;32m\"od280/od315_of_diluted_wines\"\u001b[0m\u001b[1;39m,\n", @@ -1747,12 +1476,12 @@ " \u001b[0;32m\"flavanoids\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"total_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"proline\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"proanthocyanins\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"alcalinity_of_ash\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"proanthocyanins\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"ash\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"magnesium\"\u001b[0m\u001b[1;39m,\n", - " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"malic_acid\"\u001b[0m\u001b[1;39m,\n", + " \u001b[0;32m\"nonflavanoid_phenols\"\u001b[0m\u001b[1;39m,\n", " \u001b[0;32m\"alcohol\"\u001b[0m\u001b[1;39m\n", " \u001b[1;39m]\u001b[0m\u001b[1;39m\n", " \u001b[1;39m}\u001b[0m\u001b[1;39m\n", @@ -1771,7 +1500,7 @@ }, { "cell_type": "code", - "execution_count": 73, + "execution_count": 64, "metadata": { "scrolled": true }, @@ -1801,7 +1530,7 @@ }, { "cell_type": "code", - "execution_count": 74, + "execution_count": 65, "metadata": {}, "outputs": [ { @@ -1842,7 +1571,7 @@ }, { "cell_type": "code", - "execution_count": 75, + "execution_count": 66, "metadata": {}, "outputs": [ { @@ -1859,7 +1588,7 @@ }, { "cell_type": "code", - "execution_count": 76, + "execution_count": 67, "metadata": {}, "outputs": [ { @@ -1877,7 +1606,7 @@ }, { "cell_type": "code", - "execution_count": 77, + "execution_count": 68, "metadata": {}, "outputs": [ { @@ -1907,7 +1636,7 @@ }, { "cell_type": "code", - "execution_count": 78, + "execution_count": 69, "metadata": {}, "outputs": [], "source": [ @@ -1917,7 +1646,7 @@ }, { "cell_type": "code", - "execution_count": 79, + "execution_count": 70, "metadata": {}, "outputs": [], "source": [ @@ -1929,7 +1658,7 @@ }, { "cell_type": "code", - "execution_count": 80, + "execution_count": 71, "metadata": {}, "outputs": [], "source": [ @@ -1941,7 +1670,7 @@ }, { "cell_type": "code", - "execution_count": 81, + "execution_count": 72, "metadata": {}, "outputs": [], "source": [ @@ -1951,7 +1680,7 @@ }, { "cell_type": "code", - "execution_count": 82, + "execution_count": 73, "metadata": {}, "outputs": [ { @@ -1960,7 +1689,7 @@ "array([7, 2, 1, 0, 4, 1, 4, 9, 6, 9])" ] }, - "execution_count": 82, + "execution_count": 73, "metadata": {}, "output_type": "execute_result" } @@ -1971,7 +1700,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 74, "metadata": {}, "outputs": [], "source": [ @@ -1983,7 +1712,7 @@ }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 75, "metadata": {}, "outputs": [], "source": [ @@ -1995,7 +1724,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 76, "metadata": {}, "outputs": [], "source": [ @@ -2004,12 +1733,12 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": 77, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp0AAAGaCAYAAABNOPgbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeZxcVZn/8c/TW9JJICtLEhJAFtkGAwRwAxlBQGSTRXAYDSOgOKDOoCjggj8EQWdxGRBEjCzKpjgYAYnsA4pAgqCiIAEDSUiArGTpLN39/P64p/Gem1q6um91VXW+b171op57z711UnXq1NP3nnuuuTsiIiIiItXUVOsKiIiIiMjgp6RTRERERKpOSaeIiIiIVJ2SThERERGpOiWdIiIiIlJ1SjpFREREpOoGTdJpZtea2cXh+QFm9lwf93OVmX0539pJozCzU8zs16nYzWzHHPe/yszektf+pPfM7AIzu6bE+uizr3fpvsrMDjKz+Tnuu899qMTM7FdmNq3W9eitdB+V/l3Nad/6fd3EDWjSaWZzzawjNOpXQ4MekffruPvD7v7WXtTnVDN7JLPtme7+tbzrVOC1rwrvQ89jnZmtrPbrDkZm9qCZLTOzIallc83skFS8XUggW0rty91/4u6H5liv0zP7H+HuL+ax/8Eu7/7C3b/u7qeHfW/UHvL87HurUD0K9UuF5NlXZf+46m0fWo9Cu3nNzIanlp1uZg8OwGt/1cx+nF7m7u939+uq/drl6lGoPyokrz6qlr+vUr9qcaTzKHcfAewNTAW+lC1QLjEYDMKXb0TPA7gJ+Gmt69VozGw74ADAgaP7ua9B3+4aUNn+YlNkZs21rkOdawY+U+tKNBL1fzIg3H3AHsBc4JBU/B/AHeG5A2cBzwN/C8uOBJ4ClgO/BfZMbbsX8CSwErgFuBm4OKw7CJifKjsJ+DnwOrAEuBzYFVgLdAGrgOWh7LU9+wnxGcAcYCkwA5iQWufAmaHOy4ErAOvD+zI8/DveM5Cfx2B4AF8BfgP8d6ot3QB0Ax3hs/088HL4vFaFxzuAU8O23wrt4uKw7JHMZ/xp4EVgcWizTWHdV4Efp8puF8q3AJeEtrU2vN7lqf3tGJ6PBK4P7fIlkoSqZ9+nAo8A/wksA/4GvD/1WqeGOq0M606p9WdRhc92LsX7i6OBZ8L37kFg11S5LwALwnvzHHBw9vMq0R4eCeuvBP4zU59fAOeE5xOA28Jn9zfg0yX+HR8Afg+8AcwDvppaV6gexfqlK4G7gNXAIaT6KkKfB1wQ2uncdJsI79HpmfbT82/9v1CH1eE1T2LjPnTXsI/l4X0/OrXuWpK+787wnj8G7BDWGcn367Xw7/8jsMcAtJvzSPrsUWHZ6cCDqTK7APeEMs8BH0qtGwv8MtT3CZJ+Id0nfCd8jm8As4EDwvLDgfXAhvA+Pp1+74Eh4f3bI7WvLUj6qS1DXPQ3r8C/s9f1oHR/lP3dTfdR1wJXhfdqJfAQsG1Yt10o25JtZ1T59xXYMdRlBUl7v6XW/ZUevfx+DuiLpX5ESBLBZ4CvhdhDwx4DtJMkla8B+5P81TotbD8EaCP5kf53oBU4IXzBNko6w7ZPk3R8w4GhwLvDulNJdSZh2bWp/bw3NOi9w+v+D/B/qbIO3AGMAiaT/AAdHtZNDl+Uyb14Xz5KkkBUnLBu6o/QYf0rsE9oA1tl21qIC3WQpwKdwKdIEsX2bJsI2zwQ2uVk4K+EH29KJJ0hfpDUD31qfz0d+vUkicxmYdu/Aqel6raBpFNuBj4JvELyIz6c5IfmraHseGD3Wn8WVfhs3/wMSfUXwM4kCdL7wvf/86EdtAFvJfkhnpD6THoSoDc/rxLtoScROzDsp+dHbjRJcjCB5AzRbJI/eNqAt4Tv72FF/h0HAf8QttsTeBU4tjf1SC27luQH9l1hP0PZOOnsJPnjawjwnvAe9bSRqC1mXyPdLlP76+lDW8P7e0H4976XJAF5a6puS4D9SL5HPwFuDusOC+/VqNB2dwXGD0S7ITnQ0PP+vJl0knx/5gH/Euq7F0k/v1tYf3N4DAN2C2XT79U/kySmLcBngUXA0EJ9Qva9B6YDl6TWnQXcHZ4X/c0r8u/scz0yn/ubv7vZthA+25Uk34chJIluz3dkO4oknWXacR6/rzcBX+Tv34V317q/0qN3j1qcXr/dzJaTHMV5CPh6at2l7r7U3TuAjwPfd/fH3L3LkzEx64C3h0cr8G133+DuPyP5i7SQ/Uh+KM5199Xuvtbdy46XCk4Bprv7k+6+DjgfeEc4pdvjMndf7u4vkyQnUwDc/WV3HxWWlzMNuN7Dt0l6x8zeDWwL3Orus4EXgH+qcDevuPv/uHtnaHeFfCO0y5eBbwMf7nutE+H06MnA+e6+0t3nAv8FfCRV7CV3/4G7dwHXkSSXW4V13cAeZtbu7gvd/Zn+1qlOFeovTgLudPd73H0DydHgduCdJEdWhgC7mVmru8919xf68LoPk/zoHRDiE4BH3f0VYF9gC3e/yN3XezL+7Qckn+dG3P1Bd/+ju3e7+x9IfjDf04c6/cLdfxP2s7ZImS+7+zp3f4jkyOOH+vA6WW8HRpD0devd/X6SZCD9Pfhfd3/c3TtJks4pYfkGkj+qdiFJ4P/i7gtzqFNvfAX4lJltkVl+JDDX3X8Uvve/JzlqfWL4Xh4PXOjua9z9zyTfvTe5+4/dfUnY9r9I2ltvx7/eSNxO/iksg9K/eRvpZz3S0r+7hdzp7v8XfgO/SPIbOKkPr5PV599Xkna1Lckfl5X8pkuN1SLpPDYkY9u6+79mGvq81PNtgc+a2fKeB8nRjgnhsSCTpL1U5PUmkfx4d/ahrhPS+3X3VSR/0U9MlVmUer6GpHPuNTObTHJU4fo+1G9TNw34tbsvDvGNYVkl5pUvEpV5iaRd9Nc4kj+c0u32JYq0LXdfE56OcPfVJInXmcBCM7vTzHbJoU71qFB/kf1edpN8RhPdfQ7wbyRHel4zs5vNrOLPK/QtN/P3xOqfSJIpCD92mb7pAv7+B0HEzPY3swfM7HUzW0HyuY2rtE6Ub6vLQtvokVdbnQDMC+9zet9l+8GQoF5Ocmr0NTO72sw2z6FOZbn7n0iS4/Myq7YF9s98fqcAW5Oc7m4hfq+j993MPmdmfzGzFWHbkfT+83wAGBbaxHYkSdT/pupV7DdvI/2sR1q5dvXm+vAbuLRYnSrUn9/Xz5McOX/czJ4xs4/lUB8ZAPU2ZVI6iZxHchpiVOoxzN1vAhYCE83MUuUnF9nnPGBykUHS5Y4svkLSEQAQroYcSzJeLC8fAX7juqK5ImbWTnIU5z1mtsjMFpEMt3ibmb2NjT/bYp91b44up/+qn0zSLiA5fTkstW7rCva9mL//tZ7ed6/alrvPdPf3kRz9fJbkSNumIvu9NJLPaAGAu9/o7j1HwR34RoF99OZzvwk4wcy2JTnleVtYPo9k/Fu6b9rM3Y8osp8bScarTXL3kSRj5Hr6rkL16GtbHZ2+YpvK2moprwCTzCz9e1FJW/2uu+9Dcqp6Z+DcCl67vy4kGaKSTmTmAQ9lPr8R7v5JklO4ncA2qfJvfv/N7ACShOdDwGh3H0Uy7KHU5/mmcNbiVpI/Zj5MMka5Z9aSUr95kT7Wo6/tKv3vH0FyKv4VkjYFxdtV1X5f3X2Ru5/h7hOATwDfsxyntpPqqbekM+0HwJnhL0Izs+Fm9gEz2wx4lKRj+LSZtZrZcSSn0Qt5nCRJvSzsY6iZvSusexXYxszaimx7E/AvZjYlTMfzdeCxcCo0Lx8lGecilTmW5FTqbiRHC6aQjBd7mOQ9fZVkrF2P10lOSfdljsxzzWx0OKX0GZIL1yAZ8H+gmU02s5Ekp4fSsnV4U+rH5xIz2ywkNucAPy5UPs3MtjKzY0InvY5koH53mc0Gk1uBD5jZwWbWSjKebR3wWzN7q5m9N3xf15KMwyz03pRtD+G062LgGmCmuy8Pqx4HVprZF8ys3cyazWwPM9u3yK42A5a6+1oz2494CEihepTrl0r5f2bWFpKSI/n7jBhPAceZ2bDw43xaZruibZXkwqA1wOdDf3sQcBTJkeCSzGzf0Ie3kiQpaxnAthqOfN9CcjFgjzuAnc3sI+Hf0xrquWv4Xv4c+Gp4r3Yh6U96bEby2/M60GJmXwHSR25fBbbLJOhZN5KcqTiFv59ah9K/eVl9qUepz7iUI8zs3aE9fg34nbvPc/fXSRLEfw7fgY8BO2Reryq/r2Z2opn1/GGwjCTB3ZT6wIZVt0mnu88i+Qv1cpJGNYdkYDLuvh44LsRLSb7APy+yny6SDnJHkitF54fyAPeTXJywyMwWF9j2XuDLJEc4FpJ8oQqO28oKicgqS06fFyvzDpK/qDVVUuWmAT/yZOzsop4HSXs5BbgU+FI4TfW5cHr6EuA3YVnBcVJF/ILkYoinSMbJ/RDA3e8h+UH7Q1h/R2a775AcKVtmZt8tsN9PkfwQv0gyZvFGkgsNymkiSVBfIWn/7yG50GiT4O7PkVxE8T8kSeFRJFMrrScZ13ZZWL4I2JKN/xiggvZwI8lFKTemtu0iSeimkFy53pOYjiyyj38FLrJkHt6vkCTNpepRsl8qYRFJX/kKyVCAM9392bDuWyRXNL9KMkbxJ5ltvwpcF+oQjQMN7+tRwPvDv/V7wEdT+y5lc5JkahnJqdQlJLMQDKSLSC4eAiAcWTyUpC9/heR9+wZJ2wE4m+SzXEQyE8ZNJH/UAMwE7ia56O8lkiQ6fXq6py9fYmZPFqqMuz9G8r2fAPwqtbzob14BfalHuf6omBtJjhgvJblg859T684gOXK9BNid5Ir7HlX7fSUZV/2Yma0iOYvwGZ0tbAw9V2aKiIhIhpl9A9ja3SsdLy4iGXV7pFNERGSgmdkuZrZnOMW9H8lQhP8tt52IlKc7EIiIiPzdZiSn1CeQDEf4L5IhNiLSTzq9LiIiIiJVp9PrIiIiIlJ1JU+vm5kOgzYwd7fypaqvo6ND7aiBtbe317wdqQ01tnpoQ6B21OjqpR1J3+lIp4iIiIhUnZJOEREREak6JZ0iIiIiUnVKOkVERESk6pR0ioiIiEjVKekUERERkapT0ikiIiIiVaekU0RERESqTkmniIiIiFSdkk4RERERqTolnSIiIiJSdUo6RURERKTqlHSKiIiISNUp6RQRERGRqlPSKSIiIiJVp6RTRERERKpOSaeIiIiIVJ2SThERERGpOiWdIiIiIlJ1SjpFREREpOqUdIqIiIhI1SnpFBEREZGqU9IpIiIiIlWnpFNEREREqk5Jp4iIiIhUnZJOEREREak6JZ0iIiIiUnUttXzxE044IYrPOOOMKH7llVeieO3atVH8k5/8JIoXLVoUxXPmzOlvFaUhdEfRKl9QsvRmPileYJSMvY+1kkZSWRsaYZNKrhfpDevOLigdqy+SRqcjnSIiIiJSdUo6RURERKTqlHSKiIiISNWZe/FRImZW1SEkL774YhRvt912/drfypUro/iZZ57p1/76a/78+VH8zW9+M4pnzZpV1dd39+wIoZro6Oioajta4S9E8TWPfTaKl77xRhSPH7tlFB+6y7Qo3mrYzlHclBn6PNTGxq/fNTeKN2+eHMVtjChQ6+I6WRfFrV1Do7i7OR4IZjRXtP9Ktbe317wdqQ31rw210B7FTlcUbwptCKrfjgY764rfPm8e2I+1XtqR9J2OdIqIiIhI1SnpFBEREZGqU9IpIiIiIlVX0zGdBx98cBTvueeeUfyXv/wlinfdddco3nvvvaP4oIMOiuKJEydG8bx586J40qTK5trr7OyM4tdffz2Kx48fX3L7//7v/47iz33ucxW9fqU2lTGdzasz49O6MmMeM6/eNTweX7e8+W9R/FpHPL/riNZxUdzRuSKKH3zhpig+ZKd4fN8WbW+N4nbi/bW8siaKfbPWKF69+aoobs2M72siLp+3ehhHVe9j8d7weHx6tdvQUIv3V846lkXxptiGoP77ou6WgX2bsvUp1xd1bl7ddlJOvbQj6Tsd6RQRERGRqlPSKSIiIiJVp6RTRERERKqupmM68zZ69OgonjJlShTPnj07ivfdd9+K9p+99/tf//rXKM6OQR0zZkwUn3XWWVF85ZVXVvT6ldpUxnTmbaM3LVv7TNy0IR631d0a/y1n3aXntuu2eBzYkq64HbU2xXMsju7cvuTr5a0exlE1WhsaaNl5N8u1oZG2Q9XrlFYPbQgarx0NdF9U6S9GudfLW720I+k7HekUERERkapT0ikiIiIiVaekU0RERESqblCN6Rxoxx9/fBTfeuutUfynP/0piv/xH/8xipcuXVqdigUa09kY1hPf13tV94IoHtUUj79rpi2Kq/3m1sM4KrWh0iptQ02ZNlRt9dCGQO0ob9kPdVPoi6R/dKRTRERERKpOSaeIiIiIVJ2SThERERGpupbyRaTHlltuGcXf+973oripKc7hL7rooiiu9hhOaQwWT23HgnXx/LEdXfH4vDEj4vtua1CaZC3oqKwNicDGfZFXeBhKfZFUSkc6RURERKTqlHSKiIiISNUp6RQRERGRqtOYzgpk752+xRZbRPGyZcui+Lnnnqt6naTxZMdNDW8dE8VbDo3H31lX5v7IzVWpljSwcm1IxxekkErHcFpX6Xu3i5SjnkhEREREqk5Jp4iIiIhUnZJOEREREak6jeks4V3velcUn3feeSXLH3vssVGcvfe6bJqa1seT4T2x5sdRvGV7fF/sLVveFsUawylZs5dfH8Xl2pAIbNwXdbdVdtxJYzilv3SkU0RERESqTkmniIiIiFSdkk4RERERqTqN6SzhiCOOiOLW1tYovu+++6L40UcfrXqdpAE1xeOgdhl5SBRv1rl1FGfvZ6z7G0tWtg0Ntwk1qok0lKbSYzKbNsRjPr01Pi6lvkj6S0c6RURERKTqlHSKiIiISNUp6RQRERGRqtOYzpT29vYoPvzww6N4/fr1UXzhhRdG8YYNG6pTMWko2VFTq5tfj+I/L58ZxfuN/pco1rgpyerw16I424b2zbQhEdi4L+puKT2mU2M4pdp0pFNEREREqk5HOkVERERyMHzSUO9a212+YAXWLd4w090PL1+y/inpFBEREclB99putj9uXK77fPbqhWV3aGaHA98BmoFr3P2yzPoDgW8DewInu/vPUuumAV8K4cXufl1Yvg9wLdAO3AV8xt37NepCSWfKueeeG8V77bVXFN99991R/Nvf/rbqdZLGs4HVUex0RfFeoz+UWS8S66ywDYkUUmnfor4oBwZNZeZDzf0lzZqBK4D3AfOBJ8xshrv/OVXsZeBU4HOZbccAFwJTSZrA7LDtMuBK4AzgMZKk83DgV/2pq8Z0ioiIiOTELN9HL+wHzHH3F919PXAzcEy6gLvPdfc/ANlz/4cB97j70pBo3gMcbmbjgc3d/Xfh6Ob1wLH9e2d0pFNEREQkFwY05X84b5yZzUrFV7v71al4IjAvFc8H9u/lvgttOzE85hdY3i9KOkVERETyYGD5n15f7O5T895pLWzSSecHPvCBKP7yl78cxW+88UYUX3TRRVWvkzQe68qMhGqOwzbbPIpbGF7lGslgozYkvZHti7x5YMcWSqIKRzrLWQBMSsXbhGW93fagzLYPhuXb9HGfRWlMp4iIiEhOrCnfRy88AexkZtubWRtwMjCjl9WdCRxqZqPNbDRwKDDT3RcCb5jZ283MgI8Cv6j4zcjYpI90ioiIiOTFDJp6efVPXty908zOJkkgm4Hp7v6MmV0EzHL3GWa2L/C/wGjgKDP7f+6+u7svNbOvkSSuABe5+9Lw/F/5+5RJv6KfV66Dkk4RERGR3PTy6GSu3P0ukmmN0su+knr+BPHp8nS56cD0AstnAXvkWc9NKukcO3ZsFH/3u9+N4ubmeDDeXXdFnx+/+93vqlMxaTDxjBMbmtdFcVvnsNJbb1LfOiksbkOdxG1IYzalL7JjOJs6S8+8We5e7NI3NRjT2TD08yciIiKSA7PaHOlsFEo6RURERHIy0HckaiRKOkVERERyMsDXETWUQZ10ZsdoZu+dvv3220fxCy+8EMXZeTtFALoz98HO3he7s6Uzipu9tep1ksZSrg11syGKm1Abksplx2yabq5edWYa01nKoE46RURERAaOVeOORIOGkk4RERGRPOhIZ0lKOkVERERyoqvXixvUSecOO+wQxfvss0/J8uecc04UZ8d4yqapk44oXutLonjMki3i8pvHY4m726pTL2kc5drQCCs4Z7NIRdoWx/O9dm4ejwXublM2VG3GwN+RqJEM6qRTREREZMBons6SlHSKiIiI5ERjOotT0ikiIiKSAwNdvV7CoEo6t9122yj+9a9/XbL8ueeeG8V33HFH7nWSxmPxbbEZsmFIZsHoKFw/Lh43ZejPXIm10B7FQ210pkSm0akNCRv3RU0b4gVdQ+J2sn5c3Fcp9akBXb1e0qBKOkVERERqSdcRFaekU0RERCQHhu69XoqSThEREZE86Or1kgZV0vnxj388iidPnlyy/EMPPRTF7roxrcCGpnhOxeYh8USbizc8F8VbtO4Wl2dodSomDSM7L2czakNSOc8kL9kxnGW3z7Eu0ns60lncoEo6RURERGrFME0OX4KSThEREZE8mI50lqKkU0RERCQHupCotIZOOt/97ndH8ac+9aka1UQaWcvKziju3Kx0+TfWvxrFE3xKFOte61JOtg1t3bp3jWoi9WTjvqiyn+im9fE8nrrXem006UqiovTOiIiIiOTBjKamfB+9e1k73MyeM7M5ZnZegfVDzOyWsP4xM9suLD/FzJ5KPbrNbEpY92DYZ8+6Lfv79jT0kU4RERGRelGL0+tm1gxcAbwPmA88YWYz3P3PqWKnAcvcfUczOxn4BnCSu/8E+EnYzz8At7v7U6ntTnH3WXnVVUc6RURERHLSZJbroxf2A+a4+4vuvh64GTgmU+YY4Lrw/GfAwWYb7fzDYduqaegjnQcccEAUjxgxomT5F154IYpXrVqVe52k8XRn5r7r9NVxTBxPGrYPmQIikYrbkAgb90UV0/UrNWcGTQN/8/WJwLxUPB/Yv1gZd+80sxXAWGBxqsxJbJys/sjMuoDbgIu9nxOaN3TSKSIiIlI/ej8OswLjzCx9ivtqd786zxcws/2BNe7+p9TiU9x9gZltRpJ0fgS4vj+vo6RTREREJA9GNSaHX+zuU0usXwBMSsXbhGWFysw3sxZgJLAktf5k4Kb0Bu6+IPx/pZndSHIav19Jp8Z0ioiIiOSg50KiAb56/QlgJzPb3szaSBLIGZkyM4Bp4fkJwP09p8rNrAn4EKnxnGbWYmbjwvNW4EjgT/TToD7S+fTTT0fxwQcfHMVLly4dyOpInVrXtiaK13S/HsVNFn9NRtlOUdzdWp16SePYQDw+fI1X1oZEoP/zana36jhSPRjoMZ1hjObZwEygGZju7s+Y2UXALHefAfwQuMHM5gBLSRLTHgcC89z9xdSyIcDMkHA2A/cCP+hvXQd10ikiIiIyYMzY+KLw6nP3u4C7Msu+knq+FjixyLYPAm/PLFsN5H7Fo5JOERERkRzoNpilKekUERERyYmSzuKs1JRLZtav+Ziktty9Llp+R0eH2lEDa29vr3k7UhtqbPXQhkDtqNHVSzsqZasdh/tJ39wj133+z/GPzy5z9XrD0JFOERERkVxUZZ7OQUNJp4iIiEhOqjBP56ChpFNEREQkB8ltMJV0FlMy6ayXMYHS2BphHI7UN7UhyYPakQyEGtx7vWHoSKeIiIhIDgzT6fUSlHSKiIiI5EGn10tS0ikiIiKSEyWdxSnpFBEREcmBAU2mMZ3FKOkUERERyYGZ0dKs1KoYvTMiIiIiuTBdvV6Ckk4RERGRHBjQbM21rkbdGjTpuJlda2YXh+cHmNlzfdzPVWb25XxrJyIiIoOeGU1Nzbk+BpMBTTrNbK6ZdZjZKjN7NSSKI/J+HXd/2N3f2ov6nGpmj2S2PdPdv5Z3nQq89jQzm21mb5jZfDP7ppnpyLOIiEgDa7LmXB+DSS2OdB7l7iOAvYGpwJeyBTaR5GsY8G/AOGB/4GDgczWtkYiIiPSZhTGdeT4Gk5r9a9x9AfArYA8AM3MzO8vMngeeD8uONLOnzGy5mf3WzPbs2d7M9jKzJ81spZndAgxNrTvIzOan4klm9nMze93MlpjZ5Wa2K3AV8I5w5HV5KPvmafoQn2Fmc8xsqZnNMLMJqXVuZmea2fOhjleY9e5WBO5+ZTgiuz68Fz8B3tWX91JERETqgBnN1pzrYzCpWdJpZpOAI4DfpxYfS3LUbzcz2wuYDnwCGAt8H5hhZkPMrA24HbgBGAP8FDi+yOs0A3cALwHbAROBm939L8CZwKPuPsLdRxXY9r3ApcCHgPFhHzdnih0J7AvsGcodFradHBLRyb18Sw4EnullWREREakzBhrTWUItTmPfbmadwArgTuDrqXWXuvtSADP7OPB9d38srLvOzC4A3g440Ap8290d+JmZnVPk9fYDJgDnuntnWPZIkbJZpwDT3f3JUKfzgWVmtp27zw1lLnP35cByM3sAmALc7e4vAxslsoWY2cdIhhqc3st6iYiISN0xTQ5fQi2SzmPd/d4i6+alnm8LTDOzT6WWtZEkkA4sCAlnj5eK7HMS8FIq4azEBODJnsDdV5nZEpKjpXPD4kWp8muAii6MMrNjSY6mHuLui/tQRxEREakDZgy6o5N5qrcLdtJJ5DzgEne/JFvIzN4DTDQzSyWek4EXCuxzHjDZzFoKJJ5eoHzaKyTJb8/rDic51b+gzHa9YmaHAz8APuDuf8xjnyIiIlIrNujGYeapno8B/wA408z2t8RwM/uAmW0GPAp0Ap82s1YzO47kNHohjwMLgcvCPoaaWc8FO68C24QxooXcBPyLmU0xsyEkQwEeS51a77MwXvQnwPHu/nh/9yciIiK11XPv9TwfvXpds8PN7Llw4fN5BdYPMbNbwvrHzGy7sHy7MJXlU+FxVWqbfczsj2Gb7/b2QulS6jbpdPdZwBnA5cAyYA5wali3HjguxEuBk4CfF9lPF3AUsCPwMmI+ycAAACAASURBVDA/lAe4n+TinUVmttGp7TAM4MvAbSSJ6w7Ayb2pf7iQaFWJC4m+DIwE7grlVpnZr3qzbxEREalHAz85fLhg+grg/cBuwIfNbLdMsdOAZe6+I/At4BupdS+4+5TwODO1/EqSPGyn8Di8z29LT13jYZEiIiIi0hfb7b6FX3hjwcl0+uxjU74/292nFltvZu8AvuruPbPnnA/g7pemyswMZR4Nc6EvArYgGUJ4h7vvkdnneOABd98lxB8GDnL3T/Tn31JvYzpFREREGpJhNOd/IdE4M5uViq9296tT8UTiC7Hnk0w/SaEy7t5pZitIrlEB2N7Mfg+8AXzJ3R8O5eentp8flvWLkk4RERGRHPSM6czZ4lJHOvtpITDZ3ZeY2T4k01ruXqXXUtIpIiIikguzWkyZtIBkesge27DxLDs9ZeaH0+sjgSVhBqB1AO4+28xeAHYO5bcps8+K1e2FRCIiIiKNxWiy5lwfvfAEsJOZbR9m4zkZmJEpMwOYFp6fANzv7m5mW4QLkTCzt5BcMPSiuy8E3jCzt4er1j8K/KK/707JI51mpquMGpi793t6gzx0dHSoHTWw9vb2mrcjtaHGVg9tCNSOGl29tKNSkttgDuzxvDBG82xgJtBMcifFZ8zsImCWu88AfgjcYGZzSGb96ZmJ50DgIjPbAHQDZ/bcGRL4V+BaoB34VXj0i06vi4iIiOSiNpPDu/tdwF2ZZV9JPV8LnFhgu9tIpoUstM9ZwB6F1vWVkk4RERGRHJjR21PimyQlnSIiIiK5qMmFRA1DSaeIiIhIDqo0ZdKgoaRTREREJA9WlcnhBw0lnSIiIiI5SI50KuksRkmniIiISC5Mp9dLUNIpIiIikhMd6SxOSaeIiIhIDizckUgKU9IpIiIikgczTElnUQ2ddH7uc5+L4vb29ijec889o/iEE04oub8rr7wyih999NEovuGGGyqtojSE7ijqYn0Ur/NlJbeeu/rxKN5lxPujuJm2KNZ9+Aaj6rahpkwbEumL7D0k1RdVh450FtfQSaeIiIhIvTCMJpR0FqOkU0RERCQnOtJZnJJOERERkRzoQqLSGirpvOWWW6K43BjNrO7u7pLrP/GJT0TxIYccEsUPPfRQFL/88ssVvb7Uhy7WRrHTFcWd3hHFw2yrKF7jr0bxNsP/IYrX+fIobrPNo7iFoZnXl0ZT6zbUnGlDIr2R7Ws0xrMadCFRKQ2VdIqIiIjUM43pLE5Jp4iIiEgOzHRHolKUdIqIiIjkRGM6i6vrpLO/YzifffbZKJ45c2YUv+Utb4nio446Kop32GGHKD7llFOi+NJLL62oPlIbTZ3xSKX1LfF4vGFvDI/iu1Z+J4qnjv9gFA+xkVG8qnNJFC9d+1QU77xZPDY4O6ZTGk92TGcbo6L4/leq24Y0prMxZfui7pZ4VGXLGxuiuHPz1qrXSfKmC4lKqeukU0RERKRRmC4kKklJp4iIiEhOdCFRcUo6RURERHKh0+ul1FXSOXXq1Cj+4Ac/WKRk4plnnonio48+OooXL14cxatWrYritrb4fsa/+93vovhtb3tbFI8dO7ZkfaQ+rW1ZEcXd3hnF3hxfaXjYhAviHWQmr/PM5HYjWidE8cTmfUttrrnwGtB64nkzs20oO+HhoRMzbaiM4Zk2NKF13yIlpZFlx3BmZfsiK9N5eOYi6Y32npmaOltefVH+zGpzIZGZHQ58B2gGrnH3yzLrhwDXA/sAS4CT3H2umb0PuAxoA9YD57r7/WGbB4HxQM/Ew4e6+2v9qWddJZ0iIiIijWvgj3RaMoj0CuB9wHzgCTOb4e5/ThU7DVjm7jua2cnAN4CTgMXAUe7+ipntAcwEJqa2O8XdZ+VVVyWdIiIiIrmoyYVE+wFz3P1FADO7GTgGSCedxwBfDc9/BlxuZubuv0+VeQZoN7Mh7r6uGhVV0ikiIiKSA6MqFxKNM7P00car3f3qVDwRmJeK5wP7Z/bxZhl37zSzFcBYkiOdPY4HnswknD8ysy7gNuBid+/XqIy6SjrHjx8fxWbxCJXsGM7DDjssihcuXFjR6332s5+N4t12261k+TvvvLOi/UttdBPPdffggu9F8X4TTozi9vZxUZwds7nxQKnSmjbEA6k2DInvy92E5t6rd5W2oaHEbaja9VEbGpy62+NkJe++qGuI7pRTfVW5I9Fid59avljfmdnuJKfcD00tPsXdF5jZZiRJ50dIxoX2mVqgiIiISA56LiTK89ELC4BJqXibsKxgGTNrAUaSXFCEmW0D/C/wUXd/oWcDd18Q/r8SuJHkNH6/KOkUERERyYXVIul8AtjJzLY3szbgZGBGpswMYFp4fgJwv7u7mY0C7gTOc/ffvPmvMGsxs3HheStwJPCnfr011NnpdREREZFGZgM8OXwYo3k2yZXnzcB0d3/GzC4CZrn7DOCHwA1mNgdYSpKYApwN7Ah8xcy+EpYdCqwGZoaEsxm4F/hBf+tqpcaEmm00S9iA2nbbbaN45cqVUbx06dJ+7f/pp5+O4j322KNk+UMOie9//MADD/Tr9avNfaMRQTXR0dFR03ZkXZmXz4wVzs5dV6ns/ZK7h2bGZbWWfoH6+JSKa29vr3kNa92GpH/qoQ1B7duR+qL+qZd2VMrb9t7N73r4x7nuc5sR+8yu9pjOgaIjnSIiIiK50B2JSlHSKSIiIpIT0+UyRSnpFBEREclN3Y8CqJm6TjpfeumlXPd37rnnRvHOO+9csvxjjz1WMpbG4M35dgDZkc4+JD6V0t0Wr8/OsdisORalrO5MpLleB4OB74tKH3FTalQNpiOdJdR10ikiIiLSKIwk7ZTClHSKiIiI5EZHOotR0ikiIiKSEx3pLG5QJ51HHnlkFF900UVR3NYWD7577bXXovj888+P4jVr1uRYO2kU2TGZQ1Zk1mfuZ9y0Pu5wrC0ef6cJJzc9ld87PdOmdORECmhZkZmXc6O+KB4b7JkxnuqLqsGw/O+9PmgM6qRTREREZGDpSGcxSjpFREREcqKr14tT0ikiIiKSC9OYzhIGddI5dWp8q9LsGM6sW265JYofeuih3OskjaelM/6aeHM8Z2Jne1xef+VKVrkxnJ6Zh9PQbfRkY02d8SjM7LyfXe1qN/VBvwHFDOqkU0RERGSgaJ7O0pR0ioiIiOTCdKaiBCWdIiIiIjnREKviBlXSefvtt0fxoYceWrL89ddfH8Vf+tKXcq+TNJ6N7mfcEp8qyc51V24ORc2FJ+XoyIgUUmlfVO6krvqigaHT68UNqqRTREREpHYMXUhUnJJOERERkRwkFxIp6SxGSaeIiIhITnR6vbiGTjrHjx8fxe985zujeMiQIVG8ePHiKL744oujeNWqVTnWThqFxbcnpqkjnjPR2+IOpHuj+xdndqC/cqViakOSR18kNWcGuvd6UQ2ddIqIiIjUEx3pLE5Jp4iIiEguTGM6S9A7IyIiIpIby/nRi1c0O9zMnjOzOWZ2XoH1Q8zslrD+MTPbLrXu/LD8OTM7rLf77IuGPtJ52223RfHYsWNLlv/xj38cxS+88ELudZLGs8ZeixcMi8MWixe0dY+IC2j8ziavw18ruT7bhlrJtCH9/S+AZ5pB1/DS87dmx4DqrG59GOgjnWbWDFwBvA+YDzxhZjPc/c+pYqcBy9x9RzM7GfgGcJKZ7QacDOwOTADuNbOdwzbl9lkx9XQiIiIiOei593qe//XCfsAcd3/R3dcDNwPHZMocA1wXnv8MONjMLCy/2d3XufvfgDlhf73ZZ8WUdIqIiIjkIu9T6wYwzsxmpR4fz7zoRGBeKp4flhUs4+6dwApgbIlte7PPijX06XURERGRuuFUY+6qxe4+Nfe91kBDJZ1HH310FO+9994lyz/44INRfOGFF+ZdJWlATZ1xj2DN8QH/DayO4mHdW2Z2UJVqSQPLjuHKtqF2Mm1IhI37ou6W0qdSNxrDqb6oDjnmAz5j6gJgUireJiwrVGa+mbUAI4ElZbYtt8+KqcmKiIiI5MVzfpT3BLCTmW1vZm0kFwbNyJSZAUwLz08A7nd3D8tPDle3bw/sBDzey31WrKGOdIqIiIjUtQE+0OnunWZ2NjATaAamu/szZnYRMMvdZwA/BG4wsznAUpIkklDuVuDPQCdwlrt3ARTaZ3/rqqRTREREJC8Df3odd78LuCuz7Cup52uBE4tsewlwSW/22V91nXRm59284IILori1tbXk9k899VQU697qAvDsurujeKfhh2ZKxOOqrCsz7qpJk+Ft6v66+ldRXK4NiRRSbgxnlnWrL6p7DjbwOWfDqOukU0RERKShKOksSkmniIiISF5qcHq9USjpFBEREcmLcs6i6jrp/OxnPxvF++67b8nyt99+exRrXk4BaO7oiuIXls2K4sUdc6P4gGHxzR66h2nclMTKtaF3jvvkANZGGkW2L+pqL31v9eY1cfnuYaXLi9S7uk46RURERBqGU4vJ4RuGkk4RERGRvCjnLEpJp4iIiEhelHQWVddJ5znnnFNR+bPPPjuKNS+nACwfOj+KD5h4ahQ/8PIPorh7dHx3WPUfstJfiuJybYhxVa6QNKRyYzizuoeoL2pIOr1eVF0nnSIiIiKNRJPDF6ekU0RERCQPjg5Jl6CkU0RERCQvSjqLGlRJ55gxY6J4w4YN/drfihUrSu4ve+/3kSNHltzfqFGjorjSMatdXfGcbV/4wheieM2aNRXtb1PRTFsUt9jwKD5o8mmZLSq8H3KZ9eX6n7Kvlrn3e/Ze8ItbnovioRZ/D9otHmDYxfoobqU9itVfbqzyNtRYlnb/JYorbUMtmTYkOanw3uq17ou62+IxqJXKvn5j9kWuMZ0lDKqkU0RERKRWDI3pLKV/f5aIiIiIiPSCjnSKiIiI5EWn14saVEnnH/7wh1z399Of/jSKFy5cGMVbbbVVFJ900km5vn45ixYtiuJLLrlkQF+/UQyx0VHcRDwWt81GxOtXx2NnmzZ0R/HqUR2ZV4jXNzEkilsysWVOMHQTv95zq34Vx4sfjeL21ng84ZStPhDF3hzvr5O4vkM64+27WuLyhu7vnFVpGypnPW9kllTWhrInqXyA21Ar8b83+/pqQ/loWlO6L9owKm6HlaY6FY+hbI638ObKxpxmNXVmxoS29G9/dUFXr5c0qJJOERERkVrSmM7ilHSKiIiI5EWn14tS0ikiIiKSF+WcRdV10nnXXXdF8THHHDOgr3/iiSf2a/vOzs4o7u7uLlIyMWPGjCieNWtWyfIPP/xw3yq2iWntjMfDeWbYUFdzPP+qDY/nZFzlS6I4O55vqMfj/bK6LR6XtYFV8etlRlZNGr53FA9vjff//LLfRvGtT18axR/ZOx7bO4IJUeyZ4XZN3fEC15wWG2lmaMn13Zl5K5sy83p2+Gvxeovb0BBKt6HsmMnOAW5DwzNtKEtjOHsnO4Yx2xdlx0h2D4/f164y73O507obvV7p4rnbaAxp5t9rmZ/IhuyLHOhW1llMI36kIiIiInXIcc/30R9mNsbM7jGz58P/C/6Fa2bTQpnnzWxaWDbMzO40s2fN7BkzuyxV/lQze93MngqP03tTHyWdIiIiInnpzvnRP+cB97n7TsB9IY6Y2RjgQmB/YD/gwlRy+p/uvguwF/AuM3t/atNb3H1KeFzTm8oo6RQRERHJgTt4t+f66KdjgOvC8+uAYwuUOQy4x92Xuvsy4B7gcHdf4+4PJP8uXw88CWzTn8rU9ZjO4447Loo///nPR3H23ufl7L777lFc6bya06dPj+K5c+eWLH/bbbdF8bPPPlvR60k+snO/ZcffZUcaZb/iwzvj+05jcfm1LfGci63EcyBuYGUUtzAsipu74/F/Q9fEY1BHDIvH003a+l1R/N6tKWmjLis7sGoQTI030Mq1oax227Lk+uy8nZW2oewY0uy94YcP6V8bknxUOg9l9rubnacz2xf1d57L7JjK5jXxdQldw+KUodIxl5tMX5T/1evjzCx9kcfV7n51L7fdyt17JhlfBGxVoMxEYF4qnh+WvcnMRgFHAd9JLT7ezA4E/gr8u7un91FQXSedIiIiIo0kh6OTWYvdfWqxlWZ2L1DoT8cvRvVyd7PKZxE1sxbgJuC77v5iWPxL4CZ3X2dmnyA5ivrecvtS0ikiIiKShxpcve7uhxRbZ2avmtl4d19oZuOB1woUWwAclIq3AR5MxVcDz7v7t1OvmZ7W5Rrgm72pq8Z0ioiIiOSivq5eB2YA08LzacAvCpSZCRxqZqPDBUSHhmWY2cXASODf0huEBLbH0cBfelMZK/UP6sthWKkf7tlZ2Wqjo6ND7aiBtbe317wdqQ01tnpoQ6B21OjqpR2Vss/ub/NHb/p1rvsc8ratZ5c6vV6KmY0FbgUmAy8BH3L3pWY2FTjT3U8P5T4GXBA2u8Tdf2Rm25CM9XwWWBfWXe7u15jZpSTJZiewFPiku5e9cEWn10VERERyksPRydyE0+AHF1g+Czg9FU8HpmfKzKfI5V3ufj5wfqX1UdIpIiIikgfdkagkJZ0iIiIiOanC1euDhpJOERERkbzU0en1eqOkU0RERCQH7rncRWjQUtIpIiIikpf+3y990FLSKSIiIpKTerp6vd4o6RQRERHJg65eL0lJp4iIiEhelHQWpaRTREREJCc6vV6ckk4RERGRPDi6kKgEJZ0iIiIiudCUSaUo6RQRERHJi06vF6WkU0RERCQPrttglqKkU0RERCQvSjqLUtIpIiIikgNHV6+XoqRTREREJA/u+IauWteibinpFBEREcmDxnSWpKRTREREJBeOd2mizmJKJp3ubgNVERm82tvb1Y6kX9SGJA9qR1J1DnQr6SxGRzpFREREcuCAd+n0ejFKOkVERETy4I7rSGdRSjpFREREcqIxncU11boCIiIiIoNCz5jOPB/9YGZjzOweM3s+/H90kXLTQpnnzWxaavmDZvacmT0VHluG5UPM7BYzm2Nmj5nZdr2pj5JOERERkVw43p3vo5/OA+5z952A+0IcMbMxwIXA/sB+wIWZ5PQUd58SHq+FZacBy9x9R+BbwDd6UxklnSIiIiJ58OT0ep6PfjoGuC48vw44tkCZw4B73H2puy8D7gEOr2C/PwMONrOys0NoTKeIiIhIDhyqcSHRODOblYqvdvere7ntVu6+MDxfBGxVoMxEYF4qnh+W9fiRmXUBtwEXe3Kfzze3cfdOM1sBjAUWl6rMoEk6zexaYL67f8nMDgCucfe39mE/VwEL3P1reddRREREBjF3yP9CosXuPrXYSjO7F9i6wKovxlVzN7NKz9ef4u4LzGwzkqTzI8D1Fe7jTQN6et3M5ppZh5mtMrNXzexaMxuR9+u4+8O9STjN7FQzeySz7ZkDkXCGQbjfMrNXzGyZmX3PzFqr/boiIiJSPQM9ptPdD3H3PQo8fgG8ambjAcL/XyuwiwXApFS8TViGu/f8fyVwI8mYz2gbM2sBRgJLytW1FmM6j3L3EcDewFTgS9kC4R8w2J1H8u/fA9iZ5P3Y6L0QERGRBlF/YzpnAD1Xo08DflGgzEzgUDMbHS4gOhSYaWYtZjYOIBwUOxL4U4H9ngDcH067l1SzC4lC9vwrkqQLM3MzO8vMngeeD8uODJfoLzez35rZnj3bm9leZvakma00s1uAoal1B5nZ/FQ8ycx+bmavm9kSM7vczHYFrgLeEY68Lg9lrzWzi1PbnhGmBFhqZjPMbEJqnZvZmWGKgeVmdkVvBtIGRwHfDQN3Xwe+C3ys0vdRRERE6kUyOXyej366DHhfyK0OCTFmNtXMrgFw96XA14AnwuOisGwISfL5B+ApkqObPwj7/SEw1szmAOdQ4Kr4Qmp2RNHMJgFHAD9PLT6W5JL9DjPbC5hOkpzNAv4ZmGFmbyUZq3s78G3gcpKrqG6iwCX7ZtYM3AHcTzIWoQuY6u5/MbMzgdPd/d1F6vhe4FKSrP8Z4D+Bm4EDU8WOBPYFNgdmA78E7jazycAfgD3d/eVib0Pm+TZmNtLdVxQpLyIiIvXKgTq6Daa7LwEOLrB8FnB6Kp5OknOly6wG9imy37XAiZXWpxZJ5+1m1gmsAO4Evp5ad2nIrjGzjwPfd/fHwrrrzOwC4O0kH2sr8O1wOPdnZnZOkdfbD5gAnOvunWHZI0XKZp0CTHf3J0OdzgeWmdl27j43lLnM3ZcDy83sAWAKcHdINEeV2PfdwGfCNs3Ap8PyYSTvjYiIiDQY3QazuFoknce6+71F1qUv2d8WmGZmn0otayNJIJ3kCvP0nxMvFdnnJOClVMJZiQnAkz2Bu68ysyUkUwXMDYsXpcqvAXp7YdQlJEnpU8A6kkPWewGv9qGeIiIiUmPurttgllBvk8Onk8h5wCXuPir1GObuNwELgYmZ8ZOTi+xzHjC5yMVJ5Y6Bv0KS/AJgZsNJ5qFaUO4fUo67d7j72e4+0d3fQnLV12x3V2sVERFpUHU2prOu1FvSmfYD4Ewz298Sw83sA2GuqEeBTuDTZtZqZsfx98v4sx4nSVIvC/sYambvCuteJRlH2VZk25uAfzGzKWY2hGQowGOpU+t9ZmYTzWxC+Le9HfgyyW2oREREpBH1jOnM8zGI1G3SGQa5nkFyodAyYA5wali3HjguxEuBk4gvSErvp4vkYqQdgZdJZto/Kay+n+QCoUVmttEs+mEYwJdJJkRdCOwAnNyb+pvZ5HBVfLEjsDsAvwVWk9xK6jx3/3Vv9i0iIiL1qO6uXq8r1otplURERESkjD0n7OS/PONbue5zu4uOml3qjkSNZFOYhF1ERERkAPigOzqZJyWdIiIiInmos3k6642SThEREZEcOJqnsxQlnSIiIiJ5cMc7lXQWUzLpNDMdI25g7t7b+8BXVUdHh9pRA2tvb695O1Ibamz10IZA7ajR1Us7KsnR5PAl6EiniIiISB4cHeksQUmniIiISC50G8xSlHSKiIiI5MB1pLMkJZ0iIiIiedCFRCUp6RQRERHJiWuezqKUdIqIiIjkQafXS1LSKSIiIpIH14VEpSjpFBEREcmBLiQqTUmniIiISB50IVFJSjpFREREcqLT68U11boCIiIiIoNCOL2e56M/zGyMmd1jZs+H/48uUm5aKPO8mU0LyzYzs6dSj8Vm9u2w7lQzez217vTe1EdHOkVERETyUH+n188D7nP3y8zsvBB/IV3AzMYAFwJTAQdmm9kMd18GTEmVmw38PLXpLe5+diWV0ZFOERERkRw4yen1PB/9dAxwXXh+HXBsgTKHAfe4+9KQaN4DHJ4uYGY7A1sCD/enMjrSKSIiIpKH6hzpHGdms1Lx1e5+dS+33crdF4bni4CtCpSZCMxLxfPDsrSTSY5spme+P97MDgT+Cvy7u8+jDCWdIiIiInnwqlxItNjdpxZbaWb3AlsXWPXFqGrubmZ9vV3SycBHUvEvgZvcfZ2ZfYLkKOp7y+1ESaeIiIhIHmowT6e7H1JsnZm9ambj3X2hmY0HXitQbAFwUCreBngwtY+3AS3uPjv1mktS5a8BvtmbumpMp4iIiEguvK6uXgdmANPC82nALwqUmQkcamajw9Xth4ZlPT4M3JTeICSwPY4G/tKbyuhIp4iIiEgO3MG7+noGuyouA241s9OAl4APAZjZVOBMdz/d3Zea2deAJ8I2F7n70tQ+PgQckdnvp83saKATWAqc2pvKKOkUERERyUOd3QYznAY/uMDyWcDpqXg6ML3IPt5SYNn5wPmV1kdJp4iIiEguXHckKkFJp4iIiEge6uxIZ71R0ikiIiKSh/q7I1FdUdIpIiIikgOvzjydg4aSThEREZGcKOksTkmniIiISC6cblfSWYySThEREZEcuEO319U8nXVFSaeIiIhITrp0pLOoTSrp3HHHHaN43LhxUfzBD34wig866KAo7u6OG9JVV10Vxb/5zW+ieM6cOX2pptQ5y9xtwjJ/1Ha0LI/i3y/5aRS/c+QZUdzVErcrwzKvqLvVbmrWsSyKs23o7WM/HsVOVxSrDW0ayvVF3S3ZdhBr6ow3KFdeynOdXi9pk0o6RURERKpJp9eLU9IpIiIikhMd6SxOSaeIiIhIDtx1er2UQZV07rHHHlF89tlnR/Fxxx0XxdkxnZXaf//9o7izszOKn3vuuSh+5JFHovgzn/lMFK9fv75f9ZF8WKa/6G6Kx8s98cb1UbzbqMOiuJ0tonjq2I9E8Sp/LX6BzJmYTlZH8eb2lihu2hBX0Fvi8XquYVl1JzvmctayMm3ISrehNf5qydcr14akMWT7Is8MzfXm+Mte6Und/o7hVF9UmE6vFzeokk4RERGRWnF0er0UJZ0iIiIiuXBNmVSCkk4RERGRHCRHOnV6vZiGSjr33HPPKD7rrLOi+KSTTorizTffvOT+FixYEMUPP/xwFP/tb3+L4s9//vNRPHv27Cjeb7/9onjMmDFRfMQRR0Tx008/HcXZeT+lOrrZkInjsbjrLJ4jcYRPiOLdR8WfY1aHv17y9YbY6Ch+Zvkvo3h4a9xuRgyfFMWdrXF92zqHRrFrrr2qK9uGPG5Dw62+2lC2vi20l6yP1EZ2DGd2Hs5aj5nsbo0rmJ33c5Psi1yn10tpqKRTREREpF45Tmd3V/mCmyglnSIiIiI50ZHO4pR0ioiIiOTAdSFRSXWddH7/+9+P4uy90cvNs3nfffdF8R//+McovuCCC6J47dq1Jff3zne+M4o/+clPRvH06dOjeMqUKVH86qvx3HpXXHFFFN92221R/Prr8bgu6Zs/rfx5FLvHpz4mb7Z3vJ64w1hvq6J4WGYeTvPK5qZ76RO3RvE+l384ije0xvO1rvcVUdzucbtv6ojH53VvVtdf64bU3za0gUwbyszD2d97o+816pQo7iLuy7JtaKj1b45iqY1yfUulYz6zfdH2l58Qxdkxmxu9Xia3auqIvxebZF+k+8NkQQAACxpJREFUMZ0l9a+nExERERHg71ev5/noDzMbY2b3mNnz4f+ji5S728yWm9kdmeXbm9ljZjbHzG4xs7awfEiI54T12/WmPko6RURERHKR3AYzz0c/nQfc5+47AfeFuJD/AD5SYPk3gG+5+47AMuC0sPw0YFlY/q1QriwlnSIiIiI56LkjUR0lnccA14Xn1wHHFqy3+33AyvQyMzPgvcDPCmyf3u/PgIND+ZJqOuBi6NB4fsHsPJinn356FGf/Pdkxj1deeWUU/8d//EcUr14d34+4UmPHjo3i5ubmKP7qV78axXfffXcUb7vttv16fSks28pbVsRzGu70xD5RPHzH+HPseCoe7+YHbhbFbT4iXp95wUrnytv+f+JxU9n7Fbd2DonirpZ4fJ51xp1Q14js1zjbSelvy/7aY7PjKiq/njeiuJURRUpWRzNx39plpcerb0xtqC/K9UWrnnglijfqi16O+6LWAyfG+y8zZjPvvig772b2Xu3WVa4v2jRVYXL4cWY2KxVf7e5X93Lbrdx9YXi+CNiqgtcdCyx3954LB+YDPY1yIjAPwN07zWxFKL+41A7VQkRERERy4F6Vq9cXu/vUYivN7F5g6wKrvpipm5tl/3QZWEo6RURERHIy0Fevu/shxdaZ2atmNt7dF5rZeOC1Cna9BBhlZi3haOc2QM+tHBcAk4D5ZtYCjAzlS9I5ExEREZGc1NPV68AMYFp4Pg34RW83dHcHHgB6xmGkt0/v9wTg/lC+pJoe6TzooIOi+Nxzz43i7BjO7L3Sjz/++Ch+/PHH+1Wf7BjNSZPi+xVff/31UXzXXXdF8ejRBWcieFP233PDDTdE8fLly3tVT4mtJb7P9Ustv4viXQ48LIq//9wZUfyJveL5YDszX4v+3t84u/mGtvUFy/VobonH47Wv2zwu0JX5XmfPlmTGZdX0XEqDWJdtQ6vjNrTz8PdH8Q//eFoUn/YPP4ziNjKfWc6y83BmZcd0tjGqwlfQ8Yi+yH7XNoxsjeL2A+PflM62+H1uHxt/bvEMvPn3Rd1tpT/n7L3Tm9dljuCV7Yvi7TeFvsjD1et15DLgVjM7DXgJ+BCAmU0FznT300P8MLALMMLM5gOnuftM4AvAzWZ2MfB7oKez+yFwg5nNAZYCJ/emMjq9LiIiIpKD+SyfeY7/PO+7L5S8OKcUd18CHFxg+Szg9FR8QJHtXwT2K7B8LXBipfVR0ikiIiKSA3c/vNZ1qGc6hyIiIiIiVVfTI53ZMZRdXV1FSiY6O+MRLvvvv38Un3BCPOfYLrvsUnJ/HR0dUbzrrruWjBcvjo9wb7VVJdNdbXzv9YsvvjiKN2yI53ST3hm2Jh4/t/u6+EzC68NeiOLf/vGpKP74W+JxR9nxfcs2/C2KV22I28GOw+ILB+9dGM8PO6w1rt97WuL5Z7uHxH/7bWiPx3wOWRGPhNqwIh7P17nTsChuIh5HJuUNIR6PnR3Dudyfj+JsGzrtH+L9VbsNvXPcJymlm7gNNdFWpnzc96gN9c3/b+/eYaPMrjiAf+MHtrGdeO1FSYGIvJtoFRqKJI4id2irSFCkCQV5FBRUCCgshOgoEQShRKJESIQiEnQpMVKElGoLJNBWeaCsvGLFLo8B23g8k2Krc53Mp2HmeuYbfr/uj2bEh+dw5/jq6N7RN/E7bGQj5q359p9Da6TLoc0S6Uzl+PP4uadr0fZU/I5Ozx1N16Lmj77b3QMy9Ox0AgCQnaYTAIDsNJ0AAGTX15nOe/fuhby6uhryp5/GOacDBw6EfO3atZDLziVNZ0bTmdIyZTOczWY8m+vu3bshnzp1KuS1tbWC7m3vjZ/jaD3O/j74z59D/uzv8TzUPx38fcjLHx0NeXH2FyFPjsV7tJ+34szo4ly86/3DycWQm+vxd71acr/xdjKP16gndb0Y5/tGdpy+R6+V1dD1g78JOXcNlUlrqGym0wxnb5StRaXvn+7sO6lb6Tmd6VqUatSTM4YX855Hy/Cx0wkAQHaaTgAAstN0AgCQXa3dHGStll6kurvm5uJ9wefOnQt5eXk55GfPnoX85MmTkCcmJkI+dOhQyEtLO2566sj169dDPn/+fMi7fbd6q9XtTb29sb6+3ucrd+Os7ZvWVyH/4W+/C/nj78fzXX+6/5chv3j7Zcg/f/Gr+NfNx/uT07Pv0rPuPj/zl5A/+eOvQ96ciz++svm8Xpuamup7HQ17Df1k7rfdPmBbnZ7b2WuDUENFMQh1lNfYv+vxD3q8Fm3N9Xf2d1DqiHdnpxMAgOw0nQAAZKfpBAAgu4Ge6czt5s2bIR8/frzt61+9ehXy2bNnQ75x40bIZXfJ52amc3eMfx3n5bbm4/G3jWIz5JfNf4X8zeYXIf9wKp5P2+8f3iDMUQ17De0UZ0g7raGPkxrqt0GooaIY/jrauRZ1N7ubfmj9/uENSh3x7ux0AgCQnaYTAIDsNJ0AAGTX17vXd9vKykrIx44d6+j9J0+eDPn27dtdPxPVk046Nyfifclj9TiPtzHzJuT5kU9invpxyP2em2IQxP2ARtFZDfF+KF+L4t3vjZnOvvKtRfSanU4AALLTdAIAkJ2mEwCA7IZ6pvPEiRMhX7hwIeSxsfb//EePHoV8586d3jwYlbLnq3hGYuPr9fiCA7Mhbk/GuppqLsTXJ7/qmZuizGSxUP4ihl63a1Etjptbi9h1djoBAMhO0wkAQHaaTgAAshuqmc6lpaWQL1++HPLMzEzb99fr9ZDTczk3N+M8DcOpWWyF3Pgg3l+8sS/WwcR2cqZibSPkWi2enTdSjHf7iAy4tIbSz/xt8TLkPcV3Qt4ukhoq1BA716LmvomQa9txKrOV3lTu5nL6zE4nAADZaToBAMhO0wkAQHZDNdN55MiRkGdnZ//PK7/1+vXrkI8ePRrygwcPevNgVMpYI/63GHsR5/P2LMTZ4K3ReC/2eGs65B1zVQy9spnLPUWsoUYR16KxItYQ76eRRpzRTNeitwtxxrM1Gheb9G52axH9ZqcTAIDsNJ0AAGSn6QQAILtKz3SmM5srKysdvf/WrVsh379/v9tHooLSuaf0rLvGbPxvslF8E/JoEc/KMzdFmbIa4v3U6VpUxlrEoLHTCQBAdppOAACy03QCAJBdpWY607vTHz9+HPL4ePuz8R4+fBjy6dOne/NgVNpa47OQ900cDHm8Eeftpjc+CLk5GX93S8ayeA98uZXU0HisodFiMuTJYiH7M1E96Qzm9kRcW9JzO0c3miFbixh0djoBAMhO0wkAQHaaTgAAsqvUTOfhw4dD3r9/f8itVvsJljNnzoS8sbHRmwejUtK5qJHaaMi1IsnJWXmtsTh4ZW6KshqC/yVdi5pj7Q/WtBZRdXY6AQDITtMJAEB2mk4AALKr1EznxYsXQy6b4bx06VLIq6urPX8mquf56D9D3pucmfh5/a8hfzS9HPJk7cM8D0ZlvGz9I+S9o2qIzpXNcKbSczuhalQwAADZaToBAMhO0wkAQHaVmumcn58PuVaL8zBPnz4N+erVq9mfiep723oV8g+mfxay+TvKqCGAcnY6AQDITtMJAEB2mk4AALKr1EznlStX2ub0HM+1tbXsz0T1pPN19dYXIe+tfW83H4cKUkMAnbPTCQBAdppOAACy03QCAJBdrd395bVarf3l5gy0VqvV2cW+mayvr6ujCpuamup7HamhahuEGioKdVR1g1JHvDs7nQAAZKfpBAAgO00nAADZtZ3pBACAXrDTCQBAdppOAACy03QCAJCdphMAgOw0nQAAZKfpBAAgu/8C5Adg9wGb0ocAAAAASUVORK5CYII=\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAp0AAAGaCAYAAABNOPgbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeZwcdZ3/8ddnrmSSQE4ICQlnYBFY5AjgorisICAq9/ljMboERWF1Vw5BUdgIi8d6rsohRCEsCIJiRCByiwqYoKCEQxIEkpAAOckxSeb4/P6oGqxvpbu6e6Z6pnvyfubRj/Sn6ttV3+n+ds1nvvWtb5m7IyIiIiJSTQ39XQERERERGfiUdIqIiIhI1SnpFBEREZGqU9IpIiIiIlWnpFNEREREqk5Jp4iIiIhU3YBIOs1sBzNzM2uK43vMbEoPtrOdma0xs8b8aym1Lv35m9nDZjY1x+33qF1KPsr5fsfrd+rLevWUmR1sZi8k4pfN7LActz/XzA7Ja3ubq3r73ie/A2b2YzO7PMdtX21mX8xre1J/+izpjA+IbXGDfj1uzMOqsS93/4C731Bmnd4+SLv7q+4+zN07q1GvxH67f/klH25m51VzvwNRnBiuMLNBiWWbHCjL+YWc5+dvZpeZ2U2p7ZfVLiWS9zEj/fkW+qMiXv9Sb+teiUL1iI8Hk7Je5+6Puvs/5FSHTb4z7r6Huz+cx/b7Utxu3jCzoYllU83s4T7Yd0187wvVo9w/ovP6DpjZR83st6ltn+3uX+7ttqV+9XVP54fdfRiwLzAZuCRdwCIDoge2mMQvv2Hx+/GPQBdwRz9Xra6Y2Q7AwYADR/dyW005VEnyV/KYsTlSey2pEfhMf1einqhNSZ9w9z55AC8DhyXirwN3xc8fBq4Afge0AZOA4cD1wGJgEXA50BiXbwT+B1gKvAScQ5R4NCW2NzWxr7OA54DVwLNEv8BmECV6bcAa4EJgh9R2xgMzgeXAPOCsxDYvA24Dboy3OxeY3MP35lLgob76LAbKA/hS3Ga+mWhLHwfagY3x5/rLEp/1mcCrwG8KfP4PA1cCfwDeAn4BjIrXHQIsLNTGgSPj/bfH+3s63S6J/uC7BHgFeCNuR8Pjdd31mBLXbSnwhcR+DgDmxHV6Hfhmf38WVfp8X6b4MSPru1nw/Ul+vkTHm05gffwZfS8u40THnwOBJcTHnHjdccCfE5/fRcB8YFl8LBhV5OcYCdwFvAmsiJ9PiNdtUo+4LTqwNl52Snd7Az4X12tGug3G79fFRMe4FcCPgMHxuo8Cv03Vq/tn3eQ7k37/gUHAt4HX4se3gUHJ7wJwXtyWFwMfS+znqLhOq4mO5ef3Qbu5KG4bI+JlU4GHE2V2A+6Ly7wAnJxYN5rouPEWMJvod89vE+u/AyyI1z8JHBwvz/zex+/hSmDPxLa2IjoubR3HHwKeisv9Htgr4+csux6F2lmiDZwDvAj8Ldku4uc/Bq6O36vVwCPA9unvU6JO3T/rO+J9dcb7W5nY3uWJ8mcRfX+XE32fx6fa59lx3VYC3wcsXjcprssqouPjrf19vNKjzO9nn+0oPIBNJErSvhzHDxP9ct2D6BdCM/Bz4BpgKLA10S/+T8Tlzwaej7czCniIIkkncBLRgW5/wOLGun26TnG8Q2o7vwF+AAwG9ib6pfG+eN1l8ZfqKKIk+Erg8cS2fgD8oIz3xYh+cX20vxtDvT3ig9WngP2IDrBj4+XBga3EZ31j3MZaC3z+D8dtZ8+4zB3ATfG6QyiSdCbax02p9cl2+W9x/XcChgE/A2ak6vbDuF7vBDYA74jXPwacET8fBryrvz+LKn2+yfczfczI+m4WfH+KfL5TU/tM/sKdD7w/se6nwEXx888AjwMTiJKJa4Bbivwco4ETgCHAFvF27izULgrVI9HeOoCvxvtrTbfB+P16hr8fF39H/D0gI+ks5zsDTIt/3q2JEqXfJz6L7rpNIzp2HwWsA0bG6xfz94RoJLBvX7Qbou9U98//dtJJ9F1eAHyM6PfNPkSJy+7x+p/EjyHA7nHZZNL5r/Fn2kSUaC/h78n9ZWR/76cDVyTWnQPcGz/fhyhpP5Dod8qU+GcZVOTn7HE9Um3gvri9tBZpF6uB9xK1u+90vxdkJJ0Zbe7tdga8L37f9423/b/Ab1J1uwsYAWxH9B0/Ml53C/AFoj/+BgPv6evjkx49e/T1aew7zWwl8Fuiv1L+O7Hux+4+1907iL4ARwH/4e5r3f0N4FvAqXHZk4Fvu/sCd19OlPAVMxX4mrvP9sg8d3+lVEXNbCLwbuBz7r7e3Z8CrgM+kij2W3e/26MxYjOIkgMA3P1T7v6pUvsB3gOMBW4vo6zEzOw9wPbAbe7+JFGC8P96sKnL4jbWVmT9DHd/xt3XAl8ETs7pQrPTiXrgXnL3NUQ9VKemTnH9l7u3ufvTRL0V3e2rHZhkZmPcfY27P55DfWrVJseMMr6beb0/twCnAZjZFkTHpFvidWcT9T4vdPcNRL/kTyx0itLdl7n7He6+zt1XE/U6/XMP6tMFXOruGzLa6/cSx8Uruuufg9OBae7+hru/CfwXcEZifXu8vt3d7ybq3fqHxLrdzWxLd1/h7n/MqU6lfAn4dzPbKrX8Q8DL7v4jd+9w9z8R/UF5UvzdPoHofV7n7s8CwXhMd78p/kw73P0bRAlTuWNrb+bvv8cgOmbdHD//OHCNuz/h7p0ejQPdALyr0IZ6WY+kK919eUab+pW7/yZu518A/in+DvbW6cB0d/9jvO2L423vkCjzFXdf6e6vEnUu7R0vbyc6/o+PjwHB2FGpXX2ddB7r7iPcffs4KUs28gWJ59sT/cW82MxWxr90riH6KxuiU2vJ8llJ5ESihKRS44Hl8S+J5H62TcRLEs/XAYN7MC5mCnBHnHhI+aYAv3b3pXF8c7ysUgsqWP8KUbsc04P9pI0nbLevEPVYjE0sS7ev7otozgR2BZ43s9lm9qEc6lOrCh0zSn0383p/bgaOjy9SOx74Y+IP1u2BnyeOT88RnUocm96ImQ0xs2vM7BUze4uol3ZED/54edPd15cok26v4yvcRzGF2mty28viDoNuyfZ6AlHC/oqZPWJm/5RTnTK5+zNEPWUXpVZtDxzY/dnFn9/pwDZEvbhNhO9jcIwws/PN7DkzWxW/djjlHxMeAoaY2YFxcrU30Vm97nqdl6rXRIp8hr2sR1LZx8D499TyYnWqUNCm4m0vI/t3bHebupDoLOEf4lkW/i2H+kgfqKWBw554voDoL7wxqQNZt8VEX8Zu22VsdwGwcxn7THsNGGVmWyR+uW1HdLo1F2bWSnT6/7i8trk5iN+3k4FGM+s+KA0i+kX+Tgp/rsU+66w2AJu2s3aiU0JriU6/ddepkegXVrnbfY3ol0xy2x1EYxAnZL3Q3V8ETosvuDseuN3MRse9sZuDzO9msfenwHYyPyN3f9bMXgE+QNgjBdFx5d/c/Xdl1Pc8oh6oA919iZntDfyJ6JdmyXqUW99Yur2+Fj9Pt9dtKtx2d3udW2Dbmdx9NnCMmTUD5xKNf82jp6wclwJ/BL6RWLYAeMTd358uHH+PO4i+g3+NF09MrD+YKOE5FJjr7l1mtoIyP0t37zSz24h6oF8nGqPc3YYXEJ16v6LUD9XDevT6GBjPHjGK6LPv/gNoCNG4UogS93K3GxwD49kGRlPG71h3X0I0HrT7rNf9ZvYbd59X6rXSv2ryKnF3Xwz8GviGmW1pZg1mtrOZdZ+Sug34tJlNMLORbPqXbNJ1wPlmtl98ZfwkM+tu6K8TjakrVIcFROOWrjSzwWa2F1EPyk2FyvfQcUQD/h/KcZubg2OJepV2J+op2Jto4PqjRKdYC32uRT/rEv7VzHY3syFEY9Zuj4dT/JWoZ/uD8S/TS4gS3+T+dsiYieEW4D/NbMf4QP7fRIPhC/2RFTCzfzWzrdy9i2iAPUSnXjcLpb6bFbw/5bSJm4nGb76XaCxmt6uBK7qPJWa2lZkdU2QbWxBdLLLSzEYRJUKl6tHT9npOfFwcRXQq9NZ4+dPAHma2t5kNJhoOUMn+bgEuiX/OMUSnrkseC82sxcxON7Ph7t5OlJz0WVuNk5BbgU8nFt8F7GpmZ5hZc/zY38zeEX+3fwZcFvdQ70Y4pGoLoqT0TaDJzL4EbJlYX+p7D1GbOoWodzX5h8wPgbPjXlAzs6Hx8WWLAtvoST162qaOMrP3mFkL8GWiaxcWxMMsFhEdIxvj3sZkB8/rwIT4dYXcAnwsbpODiI6BT7j7y6UqZGYnmVn3H+criBLczeYYWM9qMumMfQRo4e9XYt4OjIvX/RCYRXQg/SPRQaIgd/8p0dimm4kGRN9J9JcaRGNBL4lPZZxf4OWnEQ2Wfo3oFMil7n5/OZW3aBLcq0sUm0I0ZrDcng6JTAF+5NHUU0u6H0RX/p5ONOvB7vHnemf8mlKfdTEziAa/LyEasP5pAHdfRXQR03VEB961RFfwdutOUJaZWaExbNPjbf8G+BtRr8G/l1mnI4G5ZraGaGD/qRnjsQaqrO9mue/Pd4jGYa4ws+8W2c8tROMvH0wM5eh+7Uzg12a2mugimwOLbOPbRBf+LI3L3VtGPS4Dbojb68lFtlvIzUR/sL9ENKzocgB3/yvRH033E10NnB4DV+g7k3Q50YwAfwb+QnTcLXfS8DOAly0aWnA20Xe0L00jungIgLhn8XCisZWvEX23uy/Qgqg3djh/nyXgFqIzbxD93rmX6I/OV4i+t8nT06W+97j7E0THi/HAPYnlc4h6775H9DtvHtHFOIX0pB7ltPdCbib6Q2k50UWb/5pYdxZwAdFp8T2I/hjs9iBRz/gSM0t+dwCIv69fJBpPu5goYT01Xa6I/YEn4u/4TOAz3sfz60rPmPIdERGRwszsq8A27t6TMeMiklDLPZ0iIiJ9ysx2M7O94lPcBxAN3fh5qdeJSGm1dCGRiIhIf9uC6JT6eKJxid8gujGEiPSSTq+LiIiISNXp9LqIiIiIVF3m6XUzUzdoHXN3K12q+tra2tSO6lhra2u/tyO1ofpWC20I1I7qXa20I+k59XSKiIiISNUp6RQRERGRqlPSKSIiIiJVp6RTRERERKpOSaeIiIiIVJ2SThERERGpOiWdIiIiIlJ1SjpFREREpOqUdIqIiIhI1SnpFBEREZGqU9IpIiIiIlWnpFNEREREqk5Jp4iIiIhUnZJOEREREak6JZ0iIiIiUnVKOkVERESk6pR0ioiIiEjVKekUERERkapT0ikiIiIiVaekU0RERESqTkmniIiIiFSdkk4RERERqTolnSIiIiJSdUo6RURERKTqlHSKiIiISNUp6RQRERGRqlPSKSIiIiJV19SfOz/xxBOD+Kyzzgri1157LYjXr18fxP/3f/8XxEuWLAniefPm9baKUhe6gmiNL8osvYVPDBcYmbH3sFZSTyprQ8NsYuZ6kXJYV3pBdqxjkdQ79XSKiIiISNUp6RQRERGRqlPSKSIiIiJVZ+7FR4mYWVWHkLz00ktBvMMOO/Rqe6tXrw7iuXPn9mp7vbVw4cIg/trXvhbEc+bMqer+3T09QqhftLW1VbUdrfL5QXzdE+cF8fK33gricaO3DuLDd5sSxGOH7BrEDamhz4NtdLj/zpeDeMvG7YK4hWEFal1cBxuCuLlzcBB3NYYDwYzGirZfqdbW1n5vR2pDvWtDTbQGsdMZxJtDG4Lqt6OBzjrDt88b+/ZjrZV2JD2nnk4RERERqTolnSIiIiJSdUo6RURERKTq+nVM56GHHhrEe+21VxA/99xzQfyOd7wjiPfdd98gPuSQQ4J42223DeIFCxYE8cSJlc2119HREcRvvvlmEI8bNy7z9d/85jeD+Pzzz69o/5XaXMZ0Nq5NjU/rTI15TO29c2g4vm5l49+C+I22cH7XYc1jgritY1UQPzz/liA+bJdwfN9WLf8QxK2E22t6bV0Q+xbNQbx2yzVB3Jwa39dAWD5vtTCOqtbH4r3l4fj0arehwRZur5QNrAjizbENQe0fi7qa+vZtSten1LGoY8vqtpNSaqUdSc+pp1NEREREqk5Jp4iIiIhUnZJOEREREam6fh3TmbeRI0cG8d577x3ETz75ZBDvv//+FW0/fe/3v/71r0GcHoM6atSoID7nnHOC+Kqrrqpo/5XaXMZ05m2TNy1d+1Tc0B6O2+pqDv+Ws67sue26LBwHtqwzbEfNDeEciyM7dszcX95qYRxVvbWhvpaed7NUGxpuO1e9Tkm10Iag/tpRXx+LKv2NUWp/eauVdiQ9p55OEREREak6JZ0iIiIiUnVKOkVERESk6gbUmM6+dsIJJwTxbbfdFsTPPPNMEP/Lv/xLEC9fvrw6FYtpTGd92Eh4X+81XYuCeERDOP6ukZYgrvabWwvjqNSGslXahhpSbajaaqENgdpR3tIf6uZwLJLeUU+niIiIiFSdkk4RERERqTolnSIiIiJSdU2li0i3rbfeOoh/8IMfBHFDQ5jDT5s2LYirPYZT6oOFU9uxaEM4f2xbZzg+b9Sw8L7bGpQmaYvaKmtDIrDpscgr7IbSsUgqpZ5OEREREak6JZ0iIiIiUnVKOkVERESk6jSmswLpe6dvtdVWQbxixYogfuGFF6peJ6k/6XFTQ5tHBfHWg8Pxd9aZuj9yY1WqJXWsVBtS/4IUUukYTuvMvne7SCk6EomIiIhI1SnpFBEREZGqU9IpIiIiIlWnMZ0Z3v3udwfxRRddlFn+2GOPDeL0vddl89SwMZwMb/a6m4J469bwvthbN70ziDWGU9KeXHljEJdqQyKw6bGoq6WyfieN4ZTeUk+niIiIiFSdkk4RERERqTolnSIiIiJSdRrTmeGoo44K4ubm5iB+4IEHgvixxx6rep2kDjWE46B2G35YEG/RsU0Qp+9nrPsbS1q6DQ218f1UE6krDdljMhvawzGf3hz2S+lYJL2lnk4RERERqTolnSIiIiJSdUo6RURERKTqNKYzobW1NYiPPPLIIN64cWMQX3rppUHc3t5enYpJXUmPmlrb+GYQP7tyVhAfMPJjQaxxU5LW5m8EcboN7Z9qQyKw6bGoqyl7TKfGcEq1qadTRERERKpOPZ0iIiIiORg6cbB3ru8qXbACG5a2z3L3I0uXrH1KOkVERERy0LW+ix2PH5PrNp+/dnHJDZrZkcB3gEbgOnf/Smr9e4FvA3sBp7r77Yl1U4BL4vByd78hXr4f8GOgFbgb+Iy792rUhZLOhAsuuCCI99lnnyC+9957g/j3v/991esk9aedtUHsdAbxPiNPTq0XCXVU2IZECqn02KJjUQ4MGkrMh5r7Ls0age8D7wcWArPNbKa7P5so9irwUeD81GtHAZcCk4mawJPxa1cAVwFnAU8QJZ1HAvf0pq4a0ykiIiKSE7N8H2U4AJjn7i+5+0bgJ8AxyQLu/rK7/xlIn/s/ArjP3ZfHieZ9wJFmNg7Y0t0fj3s3bwSO7d07o55OERERkVwY0JB/d94YM5uTiK9192sT8bbAgkS8EDiwzG0Xeu228WNhgeW9oqRTREREJA8Glv/p9aXuPjnvjfaHzTrp/OAHPxjEX/ziF4P4rbfeCuJp06ZVvU5Sf6wzNRKqMQxbbMsgbmJolWskA43akJQjfSzyxr4dWyiRKvR0lrIImJiIJ8TLyn3tIanXPhwvn9DDbRalMZ0iIiIiObGGfB9lmA3sYmY7mlkLcCows8zqzgION7ORZjYSOByY5e6LgbfM7F1mZsBHgF9U/GakbNY9nSIiIiJ5MYOGMq/+yYu7d5jZuUQJZCMw3d3nmtk0YI67zzSz/YGfAyOBD5vZf7n7Hu6+3My+TJS4Akxz9+Xx80/x9ymT7qGXV66Dkk4RERGR3JTZO5krd7+baFqj5LIvJZ7PJjxdniw3HZheYPkcYM8867lZJZ2jR48O4u9+97tB3NgYDsa7++7g8+Pxxx+vTsWkzoQzTrQ3bgjilo4h2a/erL51UljYhjoI25DGbEpPpMdwNnRkz7xZ6l7s0jP9MKazbujXn4iIiEgOzPqnp7NeKOkUERERyUlf35GonijpFBEREclJH19HVFcGdNKZHqOZvnf6jjvuGMTz588P4vS8nSIAXan7YKfvi93R1BHEjd5c9TpJfSnVhrpoD+IG1Iakcukxm6abq1edmcZ0ZhnQSaeIiIhI37Fq3JFowFDSKSIiIpIH9XRmUtIpIiIikhNdvV7cgE46d9555yDeb7/9Mst/9rOfDeL0GE/ZPHXQFsTrfVkQj1q2VVh+y3AscVdLdeol9aNUGxpmBedsFqlIy9JwvteOLcOxwF0tyoaqzej7OxLVkwGddIqIiIj0Gc3TmUlJp4iIiEhONKazOCWdIiIiIjkw0NXrGQZU0rn99tsH8a9//evM8hdccEEQ33XXXbnXSeqPhbfFZlD7oNSCkUG4cUw4bsrQn7kSaqI1iAfbyFSJVKNTGxI2PRY1tIcLOgeF7WTjmPBYpdSnH+jq9UwDKukUERER6U+6jqg4JZ0iIiIiOTB07/UsSjpFRERE8qCr1zMNqKTz4x//eBBvt912meUfeeSRIHbXjWkF2hvCORUbB4UTbS5tfyGIt2rePSzP4OpUTOpGel7ORtSGpHKeSl7SYzhLvj7Hukj51NNZ3IBKOkVERET6i2GaHD6Dkk4RERGRPJh6OrMo6RQRERHJgS4kylbXSed73vOeIP73f//3fqqJ1LOm1R1B3LFFdvm3Nr4exON97yDWvdallHQb2qZ5336qidSSTY9Flf2KbtgYzuOpe633jwZdSVSU3hkRERGRPJjR0JDvo7zd2pFm9oKZzTOziwqsH2Rmt8brnzCzHeLlp5vZU4lHl5ntHa97ON5m97qte/v21HVPp4iIiEit6I/T62bWCHwfeD+wEJhtZjPd/dlEsTOBFe4+ycxOBb4KnOLu/wf8X7ydfwTudPenEq873d3n5FVX9XSKiIiI5KTBLNdHGQ4A5rn7S+6+EfgJcEyqzDHADfHz24FDzTbZ+Gnxa6umrns6Dz744CAeNmxYZvn58+cH8Zo1a3Kvk9SfrtTcdx2+NowJ44lD9iNVQCRQcRsSYdNjUcV0/Uq/M4OGvr/5+rbAgkS8EDiwWBl37zCzVcBoYGmizClsmqz+yMw6gTuAy72XE5rXddIpIiIiUjvKH4dZgTFmljzFfa27X5vnDszsQGCduz+TWHy6uy8ysy2Iks4zgBt7sx8lnSIiIiJ5MKoxOfxSd5+csX4RMDERT4iXFSqz0MyagOHAssT6U4Fbki9w90Xx/6vN7Gai0/i9Sjo1plNEREQkB90XEvXx1euzgV3MbEczayFKIGemyswEpsTPTwQe7D5VbmYNwMkkxnOaWZOZjYmfNwMfAp6hlwZ0T+fTTz8dxIceemgQL1++vC+rIzVqQ8u6IF7X9WYQN1j4NRlhuwRxV3N16iX1o51wfPg6r6wNiUDv59XsalY/Ui3o6zGd8RjNc4FZQCMw3d3nmtk0YI67zwSuB2aY2TxgOVFi2u29wAJ3fymxbBAwK044G4H7gR/2tq4DOukUERER6TNmbHpRePW5+93A3allX0o8Xw+cVOS1DwPvSi1bC+R+xaOSThEREZEc6DaY2ZR0ioiIiORESWdxljXlkpn1aj4m6V/uXhMtv62tTe2ojrW2tvZ7O1Ibqm+10IZA7aje1Uo7yjJ20lA/5Wt75rrN/z3hD0+WuHq9bqinU0RERCQXVZmnc8BQ0ikiIiKSkyrM0zlgKOkUERERyUF0G0wlncVkJp21MiZQ6ls9jMOR2qY2JHlQO5K+0A/3Xq8b6ukUERERyYFhOr2eQUmniIiISB50ej2Tkk4RERGRnCjpLE5Jp4iIiEgODGgwjeksRkmniIiISA7MjKZGpVbF6J0RERERyYXp6vUMSjpFREREcmBAozX2dzVq1oBIx81sBzNzM2uK43vMbEoPtrOdma0xU4sRERGRCpnR0NCY62Mg6bOk08xeNrO2OKl73cx+bGbDqrEvd/+Au99QZp0OS7zuVXcf5u6d1ahXYr9bm9ktZvaama0ys9+Z2YHV3KeIiIhUX4M15voYSPq6p/PD7j4M2BeYDFySLmCRAdEDm2EYMBvYDxgF3AD8qlpJuIiIiFSfxWM683wMJP3y07j7IuAeYE8AM3vYzK4ws98B64CdzGy4mV1vZovNbJGZXd592tvMGs3sf8xsqZm9BHwwuf14e1MT8Vlm9pyZrTazZ81sXzObAWwH/DLufb2wwGn68WY208yWm9k8Mzsrsc3LzOw2M7sx3u5cM5tc5s//krt/090Xu3unu18LtAD/0Iu3VURERPqTGY3WmOtjIOmXC4nMbCJwFPCzxOIzgA8ALxCNxb0NeAOYBAwF7gIWANcAZwEfAvYB1gJ3ZOzrJOAy4FhgDrAz0O7uZ5jZwcBUd78/LrtD6uU/AZ4BxgO7AfeZ2Xx3fzBefzRwPPAx4HLge8C74m39AMDdP1XG+7E3UdI5r1RZERERqU0GA24cZp76Oum808w6gFXAr4D/Tqz7sbvPBTCzsURJ6Qh3bwPWmtm3gI8TJZ0nA9929wVx+SuBQ4rscyrwNXefHcdlJXZxYvxu4IPuvh54ysyuAz4CdCedv3X3u+PyM4D/6H59Oclm/LotgRnAf7n7qnJeIyIiIrXINDl8hr5OOo/t7lUsYEHi+fZAM7DY7O3bSTUkyoxPlX8lY58TgfmVV5XxwHJ3X53aT/IU+pLE83XAYDNrcveOcnZgZq3AL4HH3f3KHtRRREREaoSZejqz1NI8nZ54vgDYAIwpksAtJkomu22Xsd0FRKfUS+0z7TVglJltkUg8twMWZbymbGY2CLgTWAh8Io9tioiISH+yATcOM0812Qfs7ouBXwPfMLMtzazBzHY2s3+Oi9wGfNrMJpjZSOCijM1dB5xvZvvFV8ZPMrPt43WvAzsVqcMC4PfAlWY22Mz2As4Eburtz2dmzcDtQBswxd27ertNERER6V/d917P81HWfs2ONLMX4oueN8mJzGyQmd0ar3+i+xqW+ALqNjN7Kn5cnXjNfmb2l/g137XEqeeeqsmkM/YRootrngVWECVp43oDfqwAACAASURBVOJ1PwRmAU8DfyS8ICng7j8FrgBuBlYT9S6OildfCVxiZivN7PwCLz8N2IGo1/PnwKUZwwMCZnZ18sNLOYjoQqjDgZXx1fNr4gubREREpC71/eTw8cw+3ye6GHt34DQz2z1V7ExghbtPAr4FfDWxbr677x0/zk4sv4rowu1d4seRPX5buuvqnnWGWURERETKscMeW/mlN5+Q6zb/be9rnnT3olMymtk/AZe5+xFxfDFA8loRM5sVl3ksnhZyCbAV0TU0d7n7nqltjgMecvfd4vg04BB379VwwFoa0ykiIiJStwyjMf8LicaY2ZxEfG08v3e3bQkvrl4IpO9y+HYZd+8ws1XA6Hjdjmb2J+At4BJ3fzQuvzC1zW17+4Mo6RQRERHJQfeYzpwtzerp7KXFwHbuvszM9iOa2nKPKu1LSaeIiIhILsz6Y8qkRYQz+kxg05l2usssjE+vDweWeTTGcgOAuz9pZvOBXePyE0pss2K1fCGRiIiISB0xGqwx10cZZgO7mNmOZtYCnArMTJWZCUyJn58IPOjubmZbJW4xvhPRBUMvxbMIvWVm74qvWv8I8IvevjuZPZ1mpquM6pi793p6gzy0tbWpHdWx1tbWfm9HakP1rRbaEKgd1btaaUdZottg9m1/XjxG81yiWX0agenuPtfMpgFz3H0mcD0ww8zmAcuJElOA9wLTzKwd6ALOdvfl8bpPAT8GWoF74kev6PS6iIiISC76Z3L4+Jbcd6eWfSnxfD1wUoHX3QHcUWSbc4A9C63rKSWdIiIiIjkwo9xT4pslJZ0iIiIiueiXC4nqhpJOERERkRxUacqkAUNJp4iIiEgerCqTww8YSjpFREREchD1dCrpLEZJp4iIiEguTKfXMyjpFBEREcmJejqLU9IpIiIikgOL70gkhSnpFBEREcmDGaaks6i6TjrPP//8IG5tbQ3ivfbaK4hPPPHEzO1dddVVQfzYY48F8YwZMyqtotSFriDqZGMQb/AVma9+ee0fgni3YR8I4kZaglj34RuIqtuGGlJtSKQn0veQ1LGoOtTTWVxdJ50iIiIitcIwGlDSWYySThEREZGcqKezOCWdIiIiIjnQhUTZ6irpvPXWW4O41BjNtK6ursz1n/jEJ4L4sMMOC+JHHnkkiF999dWK9i+1oZP1Qex0BnGHtwXxEBsbxOv89SCeMPQfg3iDrwziFtsyiJsYnNq/1Jv+bkONqTYkUo70sUZjPKtBFxJlqaukU0RERKSWaUxncUo6RURERHJgpjsSZVHSKSIiIpITjeksrqaTzt6O4Xz++eeDeNasWUG80047BfGHP/zhIN55552D+PTTTw/iK6+8sqL6SP9o6AhHKm1sCsfjDXlraBDfvfo7QTx53HFBPMiGB/GajmVBvHz9U0G86xbh2OD0mE6pP+kxnS2MCOIHX6tuG9KYzvqUPhZ1NYWjKpveag/iji2bq14nyZsuJMpS00mniIiISL0wXUiUSUmniIiISE50IVFxSjpFREREcqHT61lqKumcPHlyEB933HFFSkbmzp0bxEcffXQQL126NIjXrFkTxC0t4f2MH3/88SB+5zvfGcSjR4/OrI/UpvVNq4K4yzuC2BvDKw2PGP/5cAOpyes8NbndsObxQbxt4/5ZL9dceHVoI+G8mek2lJ7w8PBtU22ohKGpNjS+ef8iJaWepcdwpqWPRVbi4OGpi6Q32Xpqaup0eR2L8mfWPxcSmdmRwHeARuA6d/9Kav0g4EZgP2AZcIq7v2xm7we+ArQAG4EL3P3B+DUPA+OA7omHD3f3N3pTz5pKOkVERETqV9/3dFo0iPT7wPuBhcBsM5vp7s8mip0JrHD3SWZ2KvBV4BRgKfBhd3/NzPYEZgHbJl53urvPyauuSjpFREREctEvFxIdAMxz95cAzOwnwDFAMuk8Brgsfn478D0zM3f/U6LMXKDVzAa5+4ZqVFRJp4iIiEgOjKpcSDTGzJK9jde6+7WJeFtgQSJeCByY2sbbZdy9w8xWAaOJejq7nQD8MZVw/sjMOoE7gMvdvVejMmoq6Rw3blwQm4UjVNJjOI844oggXrx4cUX7O++884J49913zyz/q1/9qqLtS//oIpzr7uFFPwjiA8afFMStrWOCOD1mc9OBUtka2sOBVO2DwvtyN6C592pdpW1oMGEbqnZ91IYGpq7WMFnJ+1jUOUh3yqm+qtyRaKm7Ty5drOfMbA+iU+6HJxaf7u6LzGwLoqTzDKJxoT2mFigiIiKSg+4LifJ8lGERMDERT4iXFSxjZk3AcKILijCzCcDPgY+4+/zuF7j7ovj/1cDNRKfxe0VJp4iIiEgurD+SztnALma2o5m1AKcCM1NlZgJT4ucnAg+6u5vZCOBXwEXu/ru3fwqzJjMbEz9vBj4EPNOrt4YaO70uIiIiUs+sjyeHj8donkt05XkjMN3d55rZNGCOu88ErgdmmNk8YDlRYgpwLjAJ+JKZfSledjiwFpgVJ5yNwP3AD3tbV8saE2q2ySxhfWr77bcP4tWrVwfx8uXLe7X9p59+Ooj33HPPzPKHHRbe//ihhx7q1f6rzX2TEUH9oq2trV/bkXWmdp8aK5yeu65S6fsldw1Ojctqzt5BbXxKxbW2tvZ7Dfu7DUnv1EIbgv5vRzoW9U6ttKMs79x3d7/70Zty3eaEYfs9We0xnX1FPZ0iIiIiudAdibIo6RQRERHJielymaKUdIqIiIjkpuZHAfSbmk46X3nllVy3d8EFFwTxrrvumln+iSeeyIylPnhjvgeA9EhnHxSeSulqCden51hs1ByLUlJXKtJcrwNB3x+LsnvclBpVg6mnM0NNJ50iIiIi9cKI0k4pTEmniIiISG7U01mMkk4RERGRnKins7gBnXR+6EMfCuJp06YFcUtLOPjujTfeCOKLL744iNetW5dj7aRepMdkDlqVWp+6n3HDxvCAYy3h+DtNOLn5qfze6ak2pZ4TKaBpVWpezk2OReHYYE+N8dSxqBoMy//e6wPGgE46RURERPqWejqLUdIpIiIikhNdvV6ckk4RERGRXJjGdGYY0Enn5MnhrUrTYzjTbr311iB+5JFHcq+T1J+mjvBr4o3hnIkdrWF5/ZUraaXGcHpqHk5Dt9GTTTV0hKMw0/N+draq3dQG/Q4oZkAnnSIiIiJ9RfN0ZlPSKSIiIpIL05mKDEo6RURERHKiIVbFDaik88477wziww8/PLP8jTfeGMSXXHJJ7nWS+rPJ/YybwlMl6bnuSs2hqLnwpBT1jEghlR6LSp3U1bGob+j0enEDKukUERER6T+GLiQqTkmniIiISA6iC4mUdBajpFNEREQkJzq9XlxdJ53jxo0L4oMOOiiIBw0aFMRLly4N4ssvvzyI16xZk2PtpF5YeHtiGtrCORO9JTyAdG1y/+LUBvRXrlRMbUjyOBZJvzMD3Xu9qLpOOkVERERqiXo6i1PSKSIiIpIL05jODHpnRERERHJjOT/K2KPZkWb2gpnNM7OLCqwfZGa3xuufMLMdEusujpe/YGZHlLvNnqjrns477rgjiEePHp1Z/qabbgri+fPn514nqT/r7I1wwZAwbLJwQUvXsLCAxu9s9tr8jcz16TbUTKoN6e9/ATzVDDqHZs/fmh4DqrO6taGvezrNrBH4PvB+YCEw28xmuvuziWJnAivcfZKZnQp8FTjFzHYHTgX2AMYD95vZrvFrSm2zYjrSiYiIiOSg+97ref4rwwHAPHd/yd03Aj8BjkmVOQa4IX5+O3ComVm8/CfuvsHd/wbMi7dXzjYrpqRTREREJBd5n1o3gDFmNifx+Hhqp9sCCxLxwnhZwTLu3gGsAkZnvLacbVasrk+vi4iIiNQMpxpzVy1198m5b7Uf1FXSefTRRwfxvvvum1n+4YcfDuJLL7007ypJHWroCI8I1hh2+LezNoiHdG2d2kBVqiV1LD2GK92GWkm1IRE2PRZ1NWWfSt1kDKeORTXIMe/zGVMXARMT8YR4WaEyC82sCRgOLCvx2lLbrJiarIiIiEhePOdHabOBXcxsRzNrIbowaGaqzExgSvz8ROBBd/d4+anx1e07ArsAfyhzmxWrq55OERERkZrWxx2d7t5hZucCs4BGYLq7zzWzacAcd58JXA/MMLN5wHKiJJK43G3As0AHcI67dwIU2mZv66qkU0RERCQvfX96HXe/G7g7texLiefrgZOKvPYK4IpyttlbNZ10pufd/PznPx/Ezc3Nma9/6qmnglj3VheA5zfcG8S7DD08VSIcV2WdqXFXDZoMb3P317X3BHGpNiRSSKkxnGnWpWNRzXOwvs8560ZNJ50iIiIidUVJZ1FKOkVERETy0g+n1+uFkk4RERGRvCjnLKqmk87zzjsviPfff//M8nfeeWcQa15OAWhs6wzi+SvmBPHStpeD+OAh4c0euoZo3JSESrWhg8Z8sg9rI/UifSzqbM2+t3rjurB815Ds8iK1rqaTThEREZG64fTH5PB1Q0mniIiISF6UcxalpFNEREQkL0o6i6rppPOzn/1sReXPPffcINa8nAKwcvDCID54248G8UOv/jCIu0aGd4fV8UNW+ytBXKoNMabKFZK6VGoMZ1rXIB2L6pJOrxdV00mniIiISD3R5PDFKekUERERyYOjLukMSjpFRERE8qKks6gBlXSOGjUqiNvb23u1vVWrVmVuL33v9+HDh2dub8SIEUFc6ZjVzs5wzrbPfe5zQbxu3bqKtre5aKQliJtsaBAfst2ZqVdUeD/kEutLHX9K7i117/f0veCXNr0QxIMt/B60WjjAsJONQdxMaxDreLmpyttQfVne9VwQV9qGmlJtSHJS4b3V+/tY1NUSjkGtVHr/9Xksco3pzDCgkk4RERGR/mJoTGeW3v1ZIiIiIiJSBvV0ioiIiORFp9eLGlBJ55///Odct/fTn/40iBcvXhzEY8eODeJTTjkl1/2XsmTJkiC+4oor+nT/9WKQjQziBsKxuC02LFy/Nhw729DeFcRrR7Sl9hCub2BQEDelYkudYOgi3N8La+4J46WPBXFrcziecO+xHwxibwy310FY30Ed4es7m8Lyhu7vnFZpGyplI2+lllTWhtInqbyP21Az4c+b3r/aUD4a1mUfi9pHhO2w0lSn4jGUjeErvLGyMadpDR2pMaFNvdteTdDV65kGVNIpIiIi0p80prM4JZ0iIiIiedHp9aKUdIqIiIjkRTlnUTWddN59991BfMwxx/Tp/k866aRevb6joyOIu7q6ipSMzJw5M4jnzJmTWf7RRx/tWcU2M80d4Xg4Tw0b6mwM51+1oeGcjGt8WRCnx/MN9nC8X1qXheOy2lkT7i81smri0H2DeGhzuP0XV/w+iG97+sogPmPfcGzvMMYHsaeG2zV0hQtcc1psopHBmeu7UvNWNqTm9WzzN8L1FrahQWS3ofSYyY4+bkNDU20oTWM4y5Mew5g+FqXHSHYNDd/XzhLvc6nTupvsL7t47jYZQ5r6eS31K7Iuj0UOdCnrLKYeP1IRERGRGuS45/voDTMbZWb3mdmL8f8F/8I1sylxmRfNbEq8bIiZ/crMnjezuWb2lUT5j5rZm2b2VPyYWk59lHSKiIiI5KUr50fvXAQ84O67AA/EccDMRgGXAgcCBwCXJpLT/3H33YB9gHeb2QcSL73V3feOH9eVUxklnSIiIiI5cAfv8lwfvXQMcEP8/Abg2AJljgDuc/fl7r4CuA840t3XuftD0c/lG4E/AhN6U5maHtN5/PHHB/GFF14YxOl7n5eyxx57BHGl82pOnz49iF9++eXM8nfccUcQP//88xXtT/KRnvstPf4uPdIo/RUf2hHedxoLy69vCudcbCacA7Gd1UHcxJAgbuwKx/8NXheOQR02JBxPN3Gbdwfx+7Yh0yaHrPTAqgEwNV5fK9WG0lpt68z16Xk7K21D6TGk6XvDDx3UuzYk+ah0Hsr0dzc9T2f6WNTbeS7TYyob14XXJXQOCVOGSsdcbjbHovyvXh9jZsmLPK5192vLfO1Yd++eZHwJMLZAmW2BBYl4YbzsbWY2Avgw8J3E4hPM7L3AX4H/dPfkNgqq6aRTREREpJ7k0DuZttTdJxdbaWb3A4X+dPxCUC93N6t8FlEzawJuAb7r7i/Fi38J3OLuG8zsE0S9qO8rtS0lnSIiIiJ56Ier1939sGLrzOx1Mxvn7ovNbBzwRoFii4BDEvEE4OFEfC3wort/O7HP5LQu1wFfK6euGtMpIiIikovaunodmAlMiZ9PAX5RoMws4HAzGxlfQHR4vAwzuxwYDvxH8gVxAtvtaOC5cipjWT9QT7phpXa4p2dl6x9tbW1qR3WstbW139uR2lB9q4U2BGpH9a5W2lGW/fZ4pz92y69z3eagd27zZNbp9SxmNhq4DdgOeAU42d2Xm9lk4Gx3nxqX+zfg8/HLrnD3H5nZBKKxns8DG+J133P368zsSqJkswNYDnzS3UteuKLT6yIiIiI5yaF3MjfxafBDCyyfA0xNxNOB6akyCylyeZe7XwxcXGl9lHSKiIiI5EF3JMqkpFNEREQkJ1W4en3AUNIpIiIikpcaOr1ea5R0ioiIiOTAPZe7CA1YSjpFRERE8tL7+6UPWEo6RURERHJSS1ev1xolnSIiIiJ50NXrmZR0ioiIiORFSWdRSjpFREREcqLT68Up6RQRERHJg6MLiTIo6RQRERHJhaZMyqKkU0RERCQvOr1elJJOERERkTy4boOZRUmniIiISF6UdBalpFNEREQkB46uXs+ipFNEREQkD+54e2d/16JmKekUERERyYPGdGZS0ikiIiKSC8c7NVFnMZlJp7tbX1VEBq7W1la1I+kVtSHJg9qRVJ0DXUo6i1FPp4iIiEgOHPBOnV4vRkmniIiISB7ccfV0FqWkU0RERCQnGtNZXEN/V0BERERkQOge05nnoxfMbJSZ3WdmL8b/jyxSbkpc5kUzm5JY/rCZvWBmT8WPrePlg8zsVjObZ2ZPmNkO5dRHSaeIiIhILhzvyvfRSxcBD7j7LsADcRwws1HApcCBwAHApank9HR33zt+vBEvOxNY4e6TgG8BXy2nMko6RURERPLg0en1PB+9dAxwQ/z8BuDYAmWOAO5z9+XuvgK4Dziygu3eDhxqZiVnh9CYThEREZEcOFTjQqIxZjYnEV/r7teW+dqx7r44fr4EGFugzLbAgkS8MF7W7Udm1gncAVzu0X0+336Nu3eY2SpgNLA0qzIDIumMxxL8DWiOf/h7gJ+4+w2ZL9x0O9sBzwLD3V33sRIREZHyuUP+FxItdffJxVaa2f3ANgVWfSGsmruZVXq+/nR3X2RmWxAlnWcAN1a4jbf12el1M3vZzNrMbI2ZvW5mPzazYdXYl7t/oJyEM67TYYnXveruw/oi4TSzg8zsD2a22sz+bGbvqfY+RUREpLr6ekynux/m7nsWePwCeN3MxgHE/79RYBOLgImJeEK8DHfv/n81cDPRmM/gNWbWBAwHlpWqa1+P6fywuw8D9gUmA5ekC1hkQI81jQft/hL4OjAC+Brwy2JXlYmIiEgdqL0xnTOB7qvRpwC/KFBmFnC4mY2M85DDgVlm1mRmYwDMrBn4EPBMge2eCDwYn3bP1C/JXZw53wPsCW9fkn+Fmf0OWAfsZGbDzex6M1tsZovM7HIza4zLN5rZ/5jZUjN7Cfhgcvvx9qYm4rPM7Lm4V/FZM9vXzGYA2xEle2vM7EIz28HMPM7aMbPxZjbTzJbH0wKcldjmZWZ2m5ndGG93rpkV7f5OOQhY4u4/dfdOd78JeBM4vodvqYiIiPS7aHL4PB+99BXg/Wb2InBYHGNmk83sOgB3Xw58GZgdP6bFywYRJZ9/Bp4i6t38Ybzd64HRZjYP+CwFroovpF/GdJrZROAo4GeJxWcAHwBeAAy4jagbeBIwFLiLaNDqNcBZRBn3PsBaonEGxfZ1EnAZ0RVbc4CdgXZ3P8PMDgamuvv9cdkdUi//CVFWPx7YDbjPzOa7+4Px+qOJEsWPAZcD3wPeFW/rBwDu/qliVSsQ71ns5xAREZEa50AN3QbT3ZcBhxZYPgeYmoinA9NTZdYC+xXZ7nrgpErr09dJ551m1gGsAn4F/Hdi3Y/dfS6AmY0lSkpHuHsbsNbMvgV8nCjpPBn4trsviMtfCRxSZJ9Tga+5++w4nldORePE+N3AB+M396n4r4KPAN1J52/d/e64/AzgP7pfn5FsAjwGjDez04imGvh/RMnwkHLqJiIiIrVJt8Esrq+TzmO7exULSF6uvz3QDCxOTPvUkCgzPlX+lYx9TgTmV15VxgPL48Gzyf0kT6EvSTxfBww2syZ378jasLsvM7NjgP8Bvk80nuJ+omkKREREpA65u26DmaGWpkxK9kcvADYAY4okcIsJr7TaLmO7C4h6EUvtM+01YJSZbZFIPLcjvqKrt9z9EWB/ePvKr5eAb+SxbREREekf6uksriavEo8nMv018A0z29LMGsxsZzP757jIbcCnzWxCfKVV1gDW64DzzWy/+Mr4SWa2fbzudWCnInVYAPweuNLMBpvZXkS3fbophx8RM9vHzJrNbEuiHs8F7j4rj22LiIhIP+ge05nnYwCpyaQz9hGghWiy9hVEYx/Hxet+SHRK+mngj4QXJAXc/afAFUTzS60G7gRGxauvBC4xs5Vmdn6Bl58G7EDU6/lz4NKM4QEBM7vazK7OKHIh0cz9C+Kf67hytisiIiK1quauXq8pVsa0SiIiIiJSwl7jd/FfnvWtXLe5w7QPP5l1R6J6UktjOkVERETqmA+43sk8KekUERERyUONzdNZa5R0ioiIiOTA0dXrWZR0ioiIiOTBHe9Q0llMZtJpZuojrmPunr7VZr9oa2tTO6pjra2t/d6O1IbqWy20IVA7qne10o4yOZocPoN6OkVERETy4KinM4OSThEREZFc6DaYWZR0ioiIiOTA1dOZSUmniIiISB50IVEmJZ0iIiIiOXHN01mUkk4RERGRPOj0eiYlnSIiIiJ5cF1IlEVJp4iIiEgOdCFRNiWdIiIiInnQhUSZlHSKiIiI5ESn14tr6O8KiIiIiAwI8en1PB+9YWajzOw+M3sx/n9kkXJT4jIvmtmUeNkWZvZU4rHUzL4dr/uomb2ZWDe1nPqop1NEREQkD7V3ev0i4AF3/4qZXRTHn0sWMLNRwKXAZMCBJ81spruvAPZOlHsS+Fnipbe6+7mVVEY9nSIiIiI5cKLT63k+eukY4Ib4+Q3AsQXKHAHc5+7L40TzPuDIZAEz2xXYGni0N5VRT6eIiIhIHqrT0znGzOYk4mvd/doyXzvW3RfHz5cAYwuU2RZYkIgXxsuSTiXq2UzOfH+Cmb0X+Cvwn+6+gBKUdIqIiIjkwatyIdFSd59cbKWZ3Q9sU2DVF4KqubuZ9fR2SacCZyTiXwK3uPsGM/sEUS/q+0ptREmniIiISB76YZ5Odz+s2Doze93Mxrn7YjMbB7xRoNgi4JBEPAF4OLGNdwJN7v5kYp/LEuWvA75WTl01plNEREQkF15TV68DM4Ep8fMpwC8KlJkFHG5mI+Or2w+Pl3U7Dbgl+YI4ge12NPBcOZVRT6eIiIhIDtzBO3t6BrsqvgLcZmZnAq8AJwOY2WTgbHef6u7LzezLwOz4NdPcfXliGycDR6W2+2kzOxroAJYDHy2nMko6RURERPJQY7fBjE+DH1pg+RxgaiKeDkwvso2dCiy7GLi40voo6RQRERHJheuORBmUdIqIiIjkocZ6OmuNkk4RERGRPNTeHYlqipJOERERkRx4debpHDCUdIqIiIjkRElncUo6RURERHLhdLmSzmKUdIqIiIjkwB26vKbm6awpSjpFREREctKpns6iNqukc9KkSUE8ZsyYID7uuOOC+JBDDgnirq6wIV199dVB/Lvf/S6I582b15NqSo2z1N0mLPVHbVvTyiD+07KfBvFBw88K4s6msF0Zltqj7la7udnAiiBOt6F3jf54EDudQaw2tHkodSzqakq3g1BDR/iCUuWlNNfp9UybVdIpIiIiUk06vV6ckk4RERGRnKinszglnSIiIiI5cNfp9SwDKuncc889g/jcc88N4uOPPz6I02M6K3XggQcGcUdHRxC/8MILQfzb3/42iD/zmc8E8caNG3tVH8mHpY4XXQ3heLnZb90YxLuPOCKIW9kqiCePPiOI1/gb4Q5SZ2I6WBvEW9pOQdzQHlbQm8Lxeq5hWTUnPeZyzooSbciy29A6fz1zf6XakNSH9LHIU0NzvTH8sld6Ure3Yzh1LCpMp9eLG1BJp4iIiEh/cXR6PYuSThEREZFcuKZMyqCkU0RERCQHUU+nTq8XU1dJ51577RXE55xzThCfcsopQbzllltmbm/RokVB/Oijjwbx3/72tyC+8MILg/jJJ58M4gMOOCCIR40aFcRHHXVUED/99NNBnJ73U6qji/ZUHI7F3WDhHInDfHwQ7zEi/BzT2vzNzP0NspFBPHflL4N4aHPYboYNnRjEHc1hfVs6Bgexa669qivZhjxsQ0OtttpQur5NtGbWR/pHegxneh7O/h4z2dUcVjA97+dmeSxynV7PUldJp4iIiEitcpyOrs7SBTdTSjpFREREcqKezuKUdIqIiIjkwHUhUaaaTjqvueaaIE7fG73UPJsPPPBAEP/lL38J4s9//vNBvH79+sztHXTQQUH8yU9+MoinT58exHvvvXcQv/56OLfe97///SC+4447gvjNN8NxXdIzz6z+WRC7h6c+ttti33A94QFjo60J4iGpeTjNK5ub7pVP3BbE+33vtCBubw7na93oq4K41cN239AWjs/r2qKmv9Z1qbdtqJ1UG0rNw9nbe6PvM+L0IO4kPJal29Bg690cxdI/Sh1bKh3zmT4W7fi9E4M4PWZzk/2lcquGtvB7sVkeizSmM1PvjnQiHnMnKAAACxBJREFUIiIiAvz96vU8H71hZqPM7D4zezH+f2SRcvea2Uozuyu1fEcze8LM5pnZrWbWEi8fFMfz4vU7lFMfJZ0iIiIiuYhug5nno5cuAh5w912AB+K4kK8DZxRY/lXgW+4+CVgBnBkvPxNYES//VlyuJCWdIiIiIjnoviNRDSWdxwA3xM9vAI4tWG/3B4DVyWVmZsD7gNsLvD653duBQ+Pymfp1wMXgweH8gul5MKdOnRrE6Z8nPebxqquuCuKvf/3rQbx2bXg/4kqNHj06iBsbG4P4sssuC+J77703iLfffvte7V8KS7fyplXhnIa7zN4viIdOCj/HtqfC8W7+3i2CuMWHhetTO6x0rrwd/zccN5W+X3Fzx6Ag7mwKx+dZR3gQ6hyW/hqnD1L627K39tzi+IrKb+StIG5mWJGS1dFIeGzttOzx6ptSG+qJUseiNbNfC+JNjkWvhsei5vduG26/xJjNvI9F6Xk30/dqt85Sx6LNUxUmhx9jZnMS8bXufm2Zrx3r7ovj50uAsRXsdzSw0t27LxxYCHQ3ym2BBQDu3mFmq+LyS7M2qBYiIiIikgP3qly9vtTdJxdbaWb3A9sUWPWFVN3cLP2nS99S0ikiIiKSk76+et3dDyu2zsxeN7Nx7r7YzMYBb1Sw6WXACDNrins7JwDdt3JcBEwEFppZEzA8Lp9J50xEREREclJLV68DM4Ep8fMpwC/KfaG7O/AQ0D0OI/n65HZPBB6My2fq157OQw45JIgvuOCCIE6P4UzfK/2EE04I4j/84Q+9qk96jObEieH9im+88cYgvvvuu4N45MiCMxG8Lf3zzJgxI4hXrlxZVj0ltJ7wPtevND0exLu994ggvuaFs4L4E/uE88F2pL4Wvb2/cfrl7S0bC5br1tgUjsdr3bBlWKAz9b1Ony1Jjcvq13MpdWJDug2tDdvQrkM/EMTX/+XMID7zH68P4hZSn1nO0vNwpqXHdLYwosI9qD+iJ9LftfbhzUHc+t7wd0pHS/g+t44OP7dwBt78j0VdLdmfc/re6Y0bUj14JY9F4es3h2ORx1ev15CvALeZ2ZnAK8DJAGY2GTjb3afG8aPAbsAwM1sInOnus4DPAT8xs8uBPwHdB7vrgRlmNg9YDpxaTmV0el1EREQkBwtZOeuz/rO8776QeXFOFndfBhxaYPkcYGoiPrjI618CDiiwfD1wUqX1UdIpIiIikgN3P7K/61DLdA5FRERERKquX3s602MoOzs7i5SMdHSEI1wOPPDAID7xxHDOsd122y1ze21tbUH8jne8IzNeujTs4R47tpLprja99/rll18exO3t4ZxuUp4h68Lxc3tsCM8kvDlkfhD//i9PBfHHdwrHHaXH961o/1sQr2kP28GkIeGFg/cvDueHHdIc1u+fm8L5Z7sGhX/7tbeGYz4HrQpHQrWvCsfzdewyJIgbCMeRSWmDCMdjp8dwrvQXgzjdhs78x3B71W5DB435JFm6CNtQAy0lyofHHrWhnmn8/+3dO2yU2RUH8G/8wDa2E6+9KCkQkXcTrUJDkcRR5A5tFQmKNKEgj4KCCgGFhRAdJYIglEiUCIlQRIIuJUaKkFJtgQTaKg+UlVes2OUxYBuPZ1Jsda6T+TTMXM98w+/X/dGM+PAc7hxfHd37Jn6HjWzEvDXf/nNojXQ5tFkinakcfx4/93Qt2p6K39HpuaPpWtT80Xe7e0CGnp1OAACy03QCAJCdphMAgOz6OtN57969kFdXV0P+9NM453TgwIGQr127FnLZuaTpzGg6U1qmbIaz2Yxnc929ezfkU6dOhby2tlbQve298XMcrcfZ3wf/+XPIn/09nof6p4O/D3n5o6MhL87+IuTJsXiP9vNWnBldnIt3vX84uRhycz3+rldL7jfeTubxGvWkrhfjfN/IjtP36LWyGrp+8Dch566hMmkNlc10muHsjbK1qPT90519J3UrPaczXYtSjXpyxvBi3vNoGT52OgEAyE7TCQBAdppOAACyq7Wbg6zV0otUd9fcXLwv+Ny5cyEvLy+H/OzZs5CfPHkS8sTERMiHDh0KeWlpx01PHbl+/XrI58+fD3m371Zvtbq9qbc31tfX+3zlbpy1fdP6KuQ//O13IX/8/Xi+60/3/zLkF2+/DPnnL34V/7r5eH9yevZdetbd52f+EvInf/x1yJtz8cdXNp/Xa1NTU32vo2GvoZ/M/bbbB2yr03M7e20QaqgoBqGO8hr7dz3+QY/Xoq25/s7+Dkod8e7sdAIAkJ2mEwCA7DSdAABkN9AznbndvHkz5OPHj7d9/atXr0I+e/ZsyDdu3Ai57C753Mx07o7xr+O83NZ8PP62UWyG/LL5r5C/2fwi5B9OxfNp+/3DG4Q5qmGvoZ3iDGmnNfRxUkP9Ngg1VBTDX0c716LuZnfTD63fP7xBqSPenZ1OAACy03QCAJCdphMAgOz6evf6bltZWQn52LFjHb3/5MmTId++fbvrZ6J60knn5kS8L3msHufxNmbehDw/8knMUz8Oud9zUwyCuB/QKDqrId4P5WtRvPu9MdPZV761iF6z0wkAQHaaTgAAstN0AgCQ3VDPdJ44cSLkCxcuhDw21v6f/+jRo5Dv3LnTmwejUvZ8Fc9IbHy9Hl9wYDbE7clYV1PNhfj65Fc9c1OUmSwWyl/E0Ot2LarFcXNrEbvOTicAANlpOgEAyE7TCQBAdkM107m0tBTy5cuXQ56ZmWn7/nq9HnJ6LufmZpynYTg1i62QGx/E+4s39sU6mNhOzlSsbYRcq8Wz80aK8W4fkQGX1lD6mb8tXoa8p/hOyNtFUkOFGmLnWtTcNxFybTtOZbbSm8rdXE6f2ekEACA7TScAANlpOgEAyG6oZjqPHDkS8uzs7P955bdev34d8tGjR0N+8OBBbx6MShlrxP8WYy/ifN6ehTgbvDUa78Ueb02HvGOuiqFXNnO5p4g11CjiWjRWxBri/TTSiDOa6Vr0diHOeLZG42KT3s1uLaLf7HQCAJCdphMAgOw0nQAAZFfpmc50ZnNlZaWj99+6dSvk+/fvd/tIVFA695SeddeYjf9NNopvQh4t4ll55qYoU1ZDvJ86XYvKWIsYNHY6AQDITtMJAEB2mk4AALKr1Exnenf648ePQx4fb3823sOHD0M+ffp0bx6MSltrfBbyvomDIY834rzd9MYHITcn4+9uyVgW74Evt5IaGo81NFpMhjxZLGR/JqonncHcnohrS3pu5+hGM2RrEYPOTicAANlpOgEAyE7TCQBAdpWa6Tx8+HDI+/fvD7nVaj/BcubMmZA3NjZ682BUSjoXNVIbDblWJDk5K681FgevzE1RVkPwv6RrUXOs/cGa1iKqzk4nAADZaToBAMhO0wkAQHaVmum8ePFiyGUznJcuXQp5dXW1589E9Twf/WfIe5MzEz+v/zXkj6aXQ56sfZjnwaiMl61/hLx3VA3RubIZzlR6bidUjQoGACA7TScAANlpOgEAyK5SM53z8/Mh12pxHubp06chX716NfszUX1vW69C/sH0z0I2f0cZNQRQzk4nAADZaToBAMhO0wkAQHaVmum8cuVK25ye47m2tpb9maiedL6u3voi5L217+3m41BBagigc3Y6AQDITtMJAEB2mk4AALKrtbu/vFartb/cnIHWarU6u9g3k/X1dXVUYVNTU32vIzVUbYNQQ0WhjqpuUOqId2enEwCA7DSdAABkp+kEACC7tjOdAADQC3Y6AQDITtMJAEB2mk4AALLTdAIAkJ2mEwCA7DSdAABk91/N8r/qJFBdiQAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -2056,7 +1785,7 @@ }, { "cell_type": "code", - "execution_count": 87, + "execution_count": 78, "metadata": {}, "outputs": [ { @@ -2084,7 +1813,7 @@ }, { "cell_type": "code", - "execution_count": 88, + "execution_count": 79, "metadata": {}, "outputs": [ { @@ -2117,7 +1846,7 @@ }, { "cell_type": "code", - "execution_count": 89, + "execution_count": 80, "metadata": {}, "outputs": [ { @@ -2134,7 +1863,7 @@ }, { "cell_type": "code", - "execution_count": 90, + "execution_count": 81, "metadata": {}, "outputs": [ { @@ -2152,7 +1881,7 @@ }, { "cell_type": "code", - "execution_count": 91, + "execution_count": 82, "metadata": { "scrolled": true }, @@ -2172,7 +1901,7 @@ }, { "cell_type": "code", - "execution_count": 92, + "execution_count": 83, "metadata": { "scrolled": true }, @@ -2200,7 +1929,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 84, "metadata": { "scrolled": true }, @@ -2215,7 +1944,7 @@ }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 85, "metadata": {}, "outputs": [], "source": [ @@ -2225,7 +1954,7 @@ }, { "cell_type": "code", - "execution_count": 95, + "execution_count": 86, "metadata": {}, "outputs": [], "source": [ @@ -2235,7 +1964,7 @@ }, { "cell_type": "code", - "execution_count": 96, + "execution_count": 87, "metadata": {}, "outputs": [], "source": [ @@ -2265,7 +1994,7 @@ }, { "cell_type": "code", - "execution_count": 97, + "execution_count": 88, "metadata": {}, "outputs": [], "source": [ @@ -2274,7 +2003,7 @@ }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 89, "metadata": {}, "outputs": [ { @@ -2327,14 +2056,14 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 90, "metadata": {}, "outputs": [ { "data": { "text/html": [ "\n", - "
\n", + "
\n", "
\n", " Visualization omitted, Javascript library not loaded!
\n", " Have you run `initjs()` in this notebook? If this notebook was from another\n", @@ -2344,16 +2073,16 @@ "
\n", " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 99, + "execution_count": 90, "metadata": {}, "output_type": "execute_result" } @@ -2369,7 +2098,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 91, "metadata": {}, "outputs": [ { diff --git a/notebooks/helm_examples.ipynb b/notebooks/helm_examples.ipynb index ca12390a7c..646c62efc9 100644 --- a/notebooks/helm_examples.ipynb +++ b/notebooks/helm_examples.ipynb @@ -19,18 +19,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] @@ -39,38 +55,111 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Serve Single REST Model" + "## Serve Single Model" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: mymodel\r\n", + "LAST DEPLOYED: Mon Nov 2 11:18:38 2020\r\n", + "NAMESPACE: seldon\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], "source": [ - "!helm install mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier_rest:1.3'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], + "!helm install mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier:1.5.0-dev'" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\r\n", + "\u001b[04m\u001b[31;01m#\u001b[39;49;00m \u001b[04m\u001b[31;01mS\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mu\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mc\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m:\u001b[39;49;00m \u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mg\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01my\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m.\u001b[39;49;00m\u001b[04m\u001b[31;01mj\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\r\n", + "{\r\n", + " \u001b[34;01m\"kind\"\u001b[39;49;00m: \u001b[33m\"SeldonDeployment\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"apiVersion\"\u001b[39;49;00m: \u001b[33m\"machinelearning.seldon.io/v1\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"metadata\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mymodel\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"namespace\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"labels\"\u001b[39;49;00m: {}\r\n", + " },\r\n", + " \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mymodel\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"protocol\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"annotations\"\u001b[39;49;00m: {},\r\n", + " \u001b[34;01m\"predictors\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"default\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"graph\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"model\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n", + " },\r\n", + " \u001b[34;01m\"componentSpecs\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"containers\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"model\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mock_classifier:1.5.0-dev\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"env\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"LOG_LEVEL\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"INFO\"\u001b[39;49;00m\r\n", + " },\r\n", + " ],\r\n", + " \u001b[34;01m\"resources\"\u001b[39;49;00m: {\u001b[34;01m\"requests\"\u001b[39;49;00m:{\u001b[34;01m\"memory\"\u001b[39;49;00m:\u001b[33m\"1Mi\"\u001b[39;49;00m}},\r\n", + " }\r\n", + " ]\r\n", + " },\r\n", + " }\r\n", + " ],\r\n", + " \u001b[34;01m\"replicas\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n", + " }\r\n", + " ]\r\n", + " }\r\n", + "}\r\n" + ] + } + ], "source": [ - "!helm template mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier_rest:1.3' | pygmentize -l json" + "!helm template mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier:1.5.0-dev' | pygmentize -l json" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"mymodel-default-0-model\" successfully rolled out\r\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymodel -o jsonpath='{.items[0].metadata.name}')" ] @@ -84,7 +173,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "scrolled": true }, @@ -103,87 +192,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.0406846384836026\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.05335370865277927]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(transport=\"rest\")\n", "assert(r.success==True)\n", "print(r)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm delete mymodel" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Serve Single GRPC Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier_grpc:1.3'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm template mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier_grpc:1.3' | pygmentize -l json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymodel -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Get predictions" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"mymodel\",namespace=\"seldon\",gateway_endpoint=\"localhost:8003\",gateway=\"ambassador\", debug=True)" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -193,11 +233,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.3321428950191112]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.07014111011256721]}}}\n" + ] + } + ], "source": [ "r = sc.predict(transport=\"grpc\")\n", "print(r)" @@ -205,11 +257,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"mymodel\" uninstalled\r\n" + ] + } + ], "source": [ "!helm delete mymodel" ] @@ -223,31 +283,142 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: myabtest\r\n", + "LAST DEPLOYED: Mon Nov 2 11:19:50 2020\r\n", + "NAMESPACE: seldon\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], "source": [ "!helm install myabtest ../helm-charts/seldon-abtest" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\r\n", + "\u001b[04m\u001b[31;01m#\u001b[39;49;00m \u001b[04m\u001b[31;01mS\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mu\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mc\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m:\u001b[39;49;00m \u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mb\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mb\u001b[39;49;00m\u001b[04m\u001b[31;01m_\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m_\u001b[39;49;00m\u001b[34m2\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01m.\u001b[39;49;00m\u001b[04m\u001b[31;01mj\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\r\n", + "{\r\n", + " \u001b[34;01m\"apiVersion\"\u001b[39;49;00m: \u001b[33m\"machinelearning.seldon.io/v1alpha2\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"kind\"\u001b[39;49;00m: \u001b[33m\"SeldonDeployment\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"metadata\"\u001b[39;49;00m: {\r\n", + "\t\u001b[34;01m\"labels\"\u001b[39;49;00m: {\r\n", + "\t \u001b[34;01m\"app\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m\r\n", + "\t},\r\n", + "\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"RELEASE-NAME\"\u001b[39;49;00m\r\n", + " },\r\n", + " \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + "\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"RELEASE-NAME\"\u001b[39;49;00m,\r\n", + "\t\u001b[34;01m\"predictors\"\u001b[39;49;00m: [\r\n", + "\t {\r\n", + "\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"default\"\u001b[39;49;00m,\r\n", + "\t\t\u001b[34;01m\"replicas\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m,\r\n", + "\t\t\u001b[34;01m\"componentSpecs\"\u001b[39;49;00m: [{\r\n", + "\t\t \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + "\t\t\t\u001b[34;01m\"containers\"\u001b[39;49;00m: [\r\n", + "\t\t\t {\r\n", + " \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mock_classifier:1.5.0-dev\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"imagePullPolicy\"\u001b[39;49;00m: \u001b[33m\"IfNotPresent\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-1\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"resources\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t \u001b[34;01m\"requests\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"1Mi\"\u001b[39;49;00m\r\n", + "\t\t\t\t }\r\n", + "\t\t\t\t}\r\n", + "\t\t\t }],\r\n", + "\t\t\t\u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m20\u001b[39;49;00m\r\n", + "\t\t }},\r\n", + "\t {\r\n", + "\t\t \u001b[34;01m\"metadata\"\u001b[39;49;00m:{\r\n", + "\t\t\t\u001b[34;01m\"labels\"\u001b[39;49;00m:{\r\n", + "\t\t\t \u001b[34;01m\"version\"\u001b[39;49;00m:\u001b[33m\"v2\"\u001b[39;49;00m\r\n", + "\t\t\t}\r\n", + "\t\t }, \r\n", + "\t\t\t\u001b[34;01m\"spec\"\u001b[39;49;00m:{\r\n", + "\t\t\t \u001b[34;01m\"containers\"\u001b[39;49;00m:[\r\n", + "\t\t\t\t{\r\n", + " \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mock_classifier:1.5.0-dev\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"imagePullPolicy\"\u001b[39;49;00m: \u001b[33m\"IfNotPresent\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-2\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"resources\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t \u001b[34;01m\"requests\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"1Mi\"\u001b[39;49;00m\r\n", + "\t\t\t\t }\r\n", + "\t\t\t\t}\r\n", + "\t\t\t }\r\n", + "\t\t\t],\r\n", + "\t\t\t\u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m20\u001b[39;49;00m\r\n", + "\t\t\t\t }\r\n", + "\t\t\t\t }],\r\n", + "\t\t\u001b[34;01m\"graph\"\u001b[39;49;00m: {\r\n", + "\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"RELEASE-NAME\"\u001b[39;49;00m,\r\n", + "\t\t \u001b[34;01m\"implementation\"\u001b[39;49;00m:\u001b[33m\"RANDOM_ABTEST\"\u001b[39;49;00m,\r\n", + "\t\t \u001b[34;01m\"parameters\"\u001b[39;49;00m: [\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m:\u001b[33m\"ratioA\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"value\"\u001b[39;49;00m:\u001b[33m\"0.5\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"FLOAT\"\u001b[39;49;00m\r\n", + "\t\t\t}\r\n", + "\t\t ],\r\n", + "\t\t \u001b[34;01m\"children\"\u001b[39;49;00m: [\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-1\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"children\"\u001b[39;49;00m:[]\r\n", + "\t\t\t},\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-2\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"children\"\u001b[39;49;00m:[]\r\n", + "\t\t\t} \r\n", + "\t\t ]\r\n", + "\t\t}\r\n", + "\t }\r\n", + "\t]\r\n", + " }\r\n", + "}\r\n" + ] + } + ], "source": [ "!helm template ../helm-charts/seldon-abtest | pygmentize -l json" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"myabtest-default-0-classifier-1\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"myabtest-default-0-classifier-1\" successfully rolled out\n", + "deployment \"myabtest-default-1-classifier-2\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=myabtest -o jsonpath='{.items[0].metadata.name}')\n", "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=myabtest -o jsonpath='{.items[1].metadata.name}')" @@ -262,7 +433,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": { "scrolled": true }, @@ -281,102 +452,64 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.8562060281778412\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.11299965170860979]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(transport=\"rest\")\n", "assert(r.success==True)\n", "print(r)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm delete myabtest" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Serve GRPC AB Test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install myabtest ../helm-charts/seldon-abtest --set protocol=GRPC" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm template ../helm-charts/seldon-abtest --set protocol=GRPC | pygmentize -l json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=myabtest -o jsonpath='{.items[0].metadata.name}')\n", - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=myabtest -o jsonpath='{.items[1].metadata.name}')" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Get predictions" + "#### gRPC Request" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": { "scrolled": true }, - "outputs": [], - "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"myabtest\",namespace=\"seldon\",gateway_endpoint=\"localhost:8003\",gateway=\"ambassador\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### REST Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.45187622094165814]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.07836365822139986]}}}\n" + ] + } + ], "source": [ "r = sc.predict(transport=\"grpc\")\n", "print(r)" @@ -384,11 +517,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"myabtest\" uninstalled\r\n" + ] + } + ], "source": [ "!helm delete myabtest" ] @@ -402,31 +543,177 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: mymab\r\n", + "LAST DEPLOYED: Mon Nov 2 11:22:19 2020\r\n", + "NAMESPACE: seldon\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], "source": [ "!helm install mymab ../helm-charts/seldon-mab" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], + "execution_count": 19, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\r\n", + "\u001b[04m\u001b[31;01m#\u001b[39;49;00m \u001b[04m\u001b[31;01mS\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mu\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mc\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m:\u001b[39;49;00m \u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mb\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mb\u001b[39;49;00m\u001b[04m\u001b[31;01m.\u001b[39;49;00m\u001b[04m\u001b[31;01mj\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\r\n", + "{\r\n", + " \u001b[34;01m\"apiVersion\"\u001b[39;49;00m: \u001b[33m\"machinelearning.seldon.io/v1alpha2\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"kind\"\u001b[39;49;00m: \u001b[33m\"SeldonDeployment\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"metadata\"\u001b[39;49;00m: {\r\n", + "\t\t\u001b[34;01m\"labels\"\u001b[39;49;00m: {\u001b[34;01m\"app\"\u001b[39;49;00m:\u001b[33m\"seldon\"\u001b[39;49;00m},\r\n", + "\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"RELEASE-NAME\"\u001b[39;49;00m\r\n", + " },\r\n", + " \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + "\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"RELEASE-NAME\"\u001b[39;49;00m,\r\n", + "\t\u001b[34;01m\"predictors\"\u001b[39;49;00m: [\r\n", + "\t {\r\n", + "\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"default\"\u001b[39;49;00m,\r\n", + "\t\t\u001b[34;01m\"replicas\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m,\r\n", + "\t\t\u001b[34;01m\"componentSpecs\"\u001b[39;49;00m: [{\r\n", + "\t\t \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + "\t\t\t\u001b[34;01m\"containers\"\u001b[39;49;00m: [\r\n", + "\t\t\t {\r\n", + " \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mock_classifier:1.5.0-dev\"\u001b[39;49;00m,\t\t\t\t\r\n", + "\t\t\t\t\u001b[34;01m\"imagePullPolicy\"\u001b[39;49;00m: \u001b[33m\"IfNotPresent\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-1\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"resources\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t \u001b[34;01m\"requests\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"1Mi\"\u001b[39;49;00m\r\n", + "\t\t\t\t }\r\n", + "\t\t\t\t}\r\n", + "\t\t\t }],\r\n", + "\t\t\t\u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m20\u001b[39;49;00m\r\n", + "\t\t }},\r\n", + "\t {\r\n", + "\t\t\t\u001b[34;01m\"spec\"\u001b[39;49;00m:{\r\n", + "\t\t\t \u001b[34;01m\"containers\"\u001b[39;49;00m:[\r\n", + "\t\t\t\t{\r\n", + " \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mock_classifier:1.5.0-dev\"\u001b[39;49;00m,\t\t\t\t\t\t\t\t \r\n", + "\t\t\t\t\u001b[34;01m\"imagePullPolicy\"\u001b[39;49;00m: \u001b[33m\"IfNotPresent\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-2\"\u001b[39;49;00m,\r\n", + "\t\t\t\t\u001b[34;01m\"resources\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t \u001b[34;01m\"requests\"\u001b[39;49;00m: {\r\n", + "\t\t\t\t\t\u001b[34;01m\"memory\"\u001b[39;49;00m: \u001b[33m\"1Mi\"\u001b[39;49;00m\r\n", + "\t\t\t\t }\r\n", + "\t\t\t\t}\r\n", + "\t\t\t }\r\n", + "\t\t\t],\r\n", + "\t\t\t\u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m20\u001b[39;49;00m\r\n", + "\t\t\t}\r\n", + "\t\t},\r\n", + "\t {\r\n", + "\t\t \u001b[34;01m\"spec\"\u001b[39;49;00m:{\r\n", + "\t\t\t\u001b[34;01m\"containers\"\u001b[39;49;00m: [{\r\n", + " \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mab_epsilon_greedy:1.5.0-dev\"\u001b[39;49;00m,\t\t\t\t\t\t\t\t \t\t\t \r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"eg-router\"\u001b[39;49;00m\r\n", + "\t\t\t}],\r\n", + "\t\t\t\u001b[34;01m\"terminationGracePeriodSeconds\"\u001b[39;49;00m: \u001b[34m20\u001b[39;49;00m\r\n", + "\t\t }}\r\n", + "\t ],\r\n", + "\t\t\u001b[34;01m\"graph\"\u001b[39;49;00m: {\r\n", + "\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"eg-router\"\u001b[39;49;00m,\r\n", + "\t\t \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"ROUTER\"\u001b[39;49;00m,\r\n", + "\t\t \u001b[34;01m\"parameters\"\u001b[39;49;00m: [\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"n_branches\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"2\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"INT\"\u001b[39;49;00m\r\n", + "\t\t\t},\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"epsilon\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"0.2\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"FLOAT\"\u001b[39;49;00m\r\n", + "\t\t\t},\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"verbose\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"1\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"BOOL\"\u001b[39;49;00m\r\n", + "\t\t\t}\r\n", + "\t\t ],\r\n", + "\t\t \u001b[34;01m\"children\"\u001b[39;49;00m: [\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-1\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"children\"\u001b[39;49;00m:[]\r\n", + "\t\t\t},\r\n", + "\t\t\t{\r\n", + "\t\t\t \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"classifier-2\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"type\"\u001b[39;49;00m:\u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n", + "\t\t\t \u001b[34;01m\"children\"\u001b[39;49;00m:[]\r\n", + "\t\t\t} \r\n", + "\t\t ]\r\n", + "\t\t},\r\n", + "\t\t\u001b[34;01m\"svcOrchSpec\"\u001b[39;49;00m: {\r\n", + "\t\t\u001b[34;01m\"resources\"\u001b[39;49;00m: {\u001b[34;01m\"requests\"\u001b[39;49;00m:{\u001b[34;01m\"cpu\"\u001b[39;49;00m:\u001b[33m\"0.1\"\u001b[39;49;00m}},\r\n", + "\u001b[34;01m\"env\"\u001b[39;49;00m: [\r\n", + "{\r\n", + "\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"SELDON_LOG_MESSAGES_EXTERNALLY\"\u001b[39;49;00m,\r\n", + "\u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"false\"\u001b[39;49;00m\r\n", + "},\r\n", + "{\r\n", + "\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"SELDON_LOG_MESSAGE_TYPE\"\u001b[39;49;00m,\r\n", + "\u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"seldon.message.pair\"\u001b[39;49;00m\r\n", + "},\r\n", + "{\r\n", + "\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"SELDON_LOG_REQUESTS\"\u001b[39;49;00m,\r\n", + "\u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"false\"\u001b[39;49;00m\r\n", + "},\r\n", + "{\r\n", + "\u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"SELDON_LOG_RESPONSES\"\u001b[39;49;00m,\r\n", + "\u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"false\"\u001b[39;49;00m\r\n", + "},\r\n", + "]\r\n", + "},\r\n", + "\t\t\u001b[34;01m\"labels\"\u001b[39;49;00m: {\u001b[34;01m\"fluentd\"\u001b[39;49;00m:\u001b[33m\"true\"\u001b[39;49;00m,\u001b[34;01m\"version\"\u001b[39;49;00m:\u001b[33m\"1.5.0-dev\"\u001b[39;49;00m}\r\n", + "\t }\r\n", + "\t]\r\n", + " }\r\n", + "}\r\n" + ] + } + ], "source": [ "!helm template ../helm-charts/seldon-mab | pygmentize -l json" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"mymab-default-0-classifier-1\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"mymab-default-0-classifier-1\" successfully rolled out\n", + "deployment \"mymab-default-1-classifier-2\" successfully rolled out\n", + "deployment \"mymab-default-2-eg-router\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[0].metadata.name}')\n", "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[1].metadata.name}')\n", @@ -442,7 +729,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": { "scrolled": true }, @@ -461,116 +748,94 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.1000299187972008\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.05643175042558145]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(transport=\"rest\")\n", "assert(r.success==True)\n", "print(r)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm delete mymab" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Serve GRPC Multi-Armed Bandit" + "#### gRPC Request" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!helm install mymab ../helm-charts/seldon-mab --set protocol=GRPC" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.23579893772394123]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.0641117916962909]}}}\n" + ] + } + ], "source": [ - "!helm template ../helm-charts/seldon-mab --set protocol=GRPC | pygmentize -l json" + "r = sc.predict(transport=\"grpc\")\n", + "print(r)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": { "scrolled": true }, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[0].metadata.name}')\n", - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[1].metadata.name}')\n", - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mymab -o jsonpath='{.items[2].metadata.name}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"mymab\" uninstalled\r\n" + ] + } + ], "source": [ - "### Get predictions" + "!helm delete mymab" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"mymab\",namespace=\"seldon\",gateway_endpoint=\"localhost:8003\",gateway=\"ambassador\")" - ] - }, - { - "cell_type": "markdown", "metadata": {}, - "source": [ - "#### REST Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "r = sc.predict(transport=\"grpc\")\n", - "print(r)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, "outputs": [], - "source": [ - "!helm delete mymab" - ] + "source": [] } ], "metadata": { @@ -590,7 +855,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.6.8" }, "varInspector": { "cols": { diff --git a/notebooks/istio_example.ipynb b/notebooks/istio_example.ipynb index ec92718ef1..243b5ec48a 100644 --- a/notebooks/istio_example.ipynb +++ b/notebooks/istio_example.ipynb @@ -4,8 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Example Seldon Core Deployments using Helm\n", - "\"predictor" + "# Example Seldon Core Deployments using Helm with Istio\n", + "\n", + "Prequisites\n", + "\n", + " * [Install istio](https://istio.io/latest/docs/setup/getting-started/#download)" ] }, { @@ -19,18 +22,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] @@ -53,9 +72,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting resources/seldon-gateway.yaml\n" + ] + } + ], "source": [ "%%writefile resources/seldon-gateway.yaml\n", "apiVersion: networking.istio.io/v1alpha3\n", @@ -77,11 +104,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): error when creating \"resources/seldon-gateway.yaml\": gateways.networking.istio.io \"seldon-gateway\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create -f resources/seldon-gateway.yaml -n istio-system" ] @@ -92,248 +127,163 @@ "source": [ "Ensure the istio ingress gatewaty is port-forwarded to localhost:8004\n", "\n", - " * Istio: `kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8004:80`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "ISTIO_GATEWAY=\"localhost:8004\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Start Seldon Core\n", - "\n", - "Use the setup notebook to [Install Seldon Core](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Install-Seldon-Core) with Istio Ingress. Instructions [also online](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html)." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Serve Single Model" + " * Istio: `kubectl port-forward $(kubectl get pods -l istio=ingressgateway -n istio-system -o jsonpath='{.items[0].metadata.name}') -n istio-system 8004:8080`" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": { "scrolled": true }, "outputs": [], "source": [ - "!helm install mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier_rest:1.3'" + "ISTIO_GATEWAY=\"localhost:8004\"\n", + "VERSION=!cat ../version.txt\n", + "VERSION=VERSION[0]" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm template mymodel ../helm-charts/seldon-single-model --set 'model.image=seldonio/mock_classifier_rest:1.3' | pygmentize -l json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/mymodel-default-0-model" - ] - }, - { - "cell_type": "markdown", + "execution_count": 6, "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "### Get predictions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"mymodel\",namespace=\"seldon\",gateway_endpoint=ISTIO_GATEWAY)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### REST Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "r = sc.predict(gateway=\"istio\",transport=\"rest\")\n", - "assert(r.success==True)\n", - "print(r)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm delete mymodel" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Serve AB Test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm install myabtest ../helm-charts/seldon-abtest" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm template ../helm-charts/seldon-abtest | pygmentize -l json" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/myabtest-default-0-classifier-1\n", - "!kubectl rollout status deploy/myabtest-default-1-classifier-2" + "VERSION" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Get predictions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"myabtest\",namespace=\"seldon\",gateway_endpoint=ISTIO_GATEWAY)" + "## Start Seldon Core\n", + "\n", + "Use the setup notebook to [Install Seldon Core](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html#Install-Seldon-Core) with Istio Ingress. Instructions [also online](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html)." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### REST Request" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "r = sc.predict(gateway=\"istio\",transport=\"rest\")\n", - "assert(r.success==True)\n", - "print(r)" + "## Serve Single Model" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": { "scrolled": true }, - "outputs": [], - "source": [ - "!helm delete myabtest" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Serve Multi-Armed Bandit" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: mymodel\r\n", + "LAST DEPLOYED: Mon Nov 2 11:26:51 2020\r\n", + "NAMESPACE: seldon\r\n", + "STATUS: deployed\r\n", + "REVISION: 1\r\n", + "TEST SUITE: None\r\n" + ] + } + ], "source": [ - "!helm install mymab ../helm-charts/seldon-mab" + "!helm install mymodel ../helm-charts/seldon-single-model --set model.image=seldonio/mock_classifier:$VERSION" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\r\n", + "\u001b[04m\u001b[31;01m#\u001b[39;49;00m \u001b[04m\u001b[31;01mS\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mu\u001b[39;49;00m\u001b[04m\u001b[31;01mr\u001b[39;49;00m\u001b[04m\u001b[31;01mc\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m:\u001b[39;49;00m \u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mi\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mg\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01m-\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01ma\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01m/\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01md\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mp\u001b[39;49;00m\u001b[04m\u001b[31;01ml\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01my\u001b[39;49;00m\u001b[04m\u001b[31;01mm\u001b[39;49;00m\u001b[04m\u001b[31;01me\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\u001b[04m\u001b[31;01mt\u001b[39;49;00m\u001b[04m\u001b[31;01m.\u001b[39;49;00m\u001b[04m\u001b[31;01mj\u001b[39;49;00m\u001b[04m\u001b[31;01ms\u001b[39;49;00m\u001b[04m\u001b[31;01mo\u001b[39;49;00m\u001b[04m\u001b[31;01mn\u001b[39;49;00m\r\n", + "{\r\n", + " \u001b[34;01m\"kind\"\u001b[39;49;00m: \u001b[33m\"SeldonDeployment\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"apiVersion\"\u001b[39;49;00m: \u001b[33m\"machinelearning.seldon.io/v1\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"metadata\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mymodel\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"namespace\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"labels\"\u001b[39;49;00m: {}\r\n", + " },\r\n", + " \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"mymodel\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"protocol\"\u001b[39;49;00m: \u001b[33m\"seldon\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"annotations\"\u001b[39;49;00m: {},\r\n", + " \u001b[34;01m\"predictors\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"default\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"graph\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"model\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"type\"\u001b[39;49;00m: \u001b[33m\"MODEL\"\u001b[39;49;00m,\r\n", + " },\r\n", + " \u001b[34;01m\"componentSpecs\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"spec\"\u001b[39;49;00m: {\r\n", + " \u001b[34;01m\"containers\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"model\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"image\"\u001b[39;49;00m: \u001b[33m\"seldonio/mock_classifier:1.5.0-dev\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"env\"\u001b[39;49;00m: [\r\n", + " {\r\n", + " \u001b[34;01m\"name\"\u001b[39;49;00m: \u001b[33m\"LOG_LEVEL\"\u001b[39;49;00m,\r\n", + " \u001b[34;01m\"value\"\u001b[39;49;00m: \u001b[33m\"INFO\"\u001b[39;49;00m\r\n", + " },\r\n", + " ],\r\n", + " \u001b[34;01m\"resources\"\u001b[39;49;00m: {\u001b[34;01m\"requests\"\u001b[39;49;00m:{\u001b[34;01m\"memory\"\u001b[39;49;00m:\u001b[33m\"1Mi\"\u001b[39;49;00m}},\r\n", + " }\r\n", + " ]\r\n", + " },\r\n", + " }\r\n", + " ],\r\n", + " \u001b[34;01m\"replicas\"\u001b[39;49;00m: \u001b[34m1\u001b[39;49;00m\r\n", + " }\r\n", + " ]\r\n", + " }\r\n", + "}\r\n" + ] + } + ], "source": [ - "!helm template ../helm-charts/seldon-mab | pygmentize -l json" + "!helm template mymodel ../helm-charts/seldon-single-model --set model.image=seldonio/mock_classifier:$VERSION | pygmentize -l json" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"mymodel-default-0-model\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"mymodel-default-0-model\" successfully rolled out\n" + ] + } + ], "source": [ - "!kubectl rollout status deploy/mymab-default-0-classifier-1\n", - "!kubectl rollout status deploy/mymab-default-1-classifier-2\n", - "!kubectl rollout status deploy/mymab-default-2-eg-router" + "!kubectl rollout status deploy/mymodel-default-0-model" ] }, { @@ -345,14 +295,14 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": { "scrolled": true }, "outputs": [], "source": [ "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"mymab\",namespace=\"seldon\",gateway_endpoint=ISTIO_GATEWAY)" + "sc = SeldonClient(deployment_name=\"mymodel\",namespace=\"seldon\",gateway_endpoint=ISTIO_GATEWAY)" ] }, { @@ -364,110 +314,85 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 1\n", + " values: 0.7700289654241149\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.10464583547528208]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"istio\",transport=\"rest\")\n", "assert(r.success==True)\n", "print(r)" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!helm delete mymab" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Serve with Shadow\n", - "\n", - "We'll use a pre-packaged model server but the 'shadow' flag can be set on any predictor." + "## gRPC Request" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], - "source": [ - "!pygmentize ./resources/istio_shadow.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!kubectl apply -f ./resources/istio_shadow.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/iris-default-0-iris-default\n", - "!kubectl rollout status deploy/iris-shadow-0-iris-shadow" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], - "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"iris\",namespace=\"seldon\",gateway_endpoint=ISTIO_GATEWAY)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.12757803284172686]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.05791666150083342]}}}\n" + ] + } + ], "source": [ - "r = sc.predict(gateway=\"istio\",transport=\"rest\",shape=(1,4))\n", + "r = sc.predict(gateway=\"istio\",transport=\"grpc\")\n", "assert(r.success==True)\n", "print(r)" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The traffic should go to both the default predictor and the shadow. If desired this can be checked in istio dashboards in the same way as with the istio canary example. When shadowing only the responses from the default predictor are used." - ] - }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"mymodel\" uninstalled\r\n" + ] + } + ], "source": [ - "!kubectl delete -f ./resources/istio_shadow.yaml" + "!helm delete mymodel" ] }, { @@ -495,7 +420,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.6.8" }, "varInspector": { "cols": { diff --git a/notebooks/max_grpc_msg_size.ipynb b/notebooks/max_grpc_msg_size.ipynb index 4438b05713..193bdc5461 100644 --- a/notebooks/max_grpc_msg_size.ipynb +++ b/notebooks/max_grpc_msg_size.ipynb @@ -20,6 +20,20 @@ "```" ] }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -31,22 +45,60 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "VERSION=!cat ../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -56,65 +108,44 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": { "scrolled": true }, "outputs": [], "source": [ - "%%writefile resources/model_long_timeouts.json\n", - "{\n", - " \"apiVersion\": \"machinelearning.seldon.io/v1alpha2\",\n", - " \"kind\": \"SeldonDeployment\",\n", - " \"metadata\": {\n", - " \"labels\": {\n", - " \"app\": \"seldon\"\n", - " },\n", - " \"name\": \"model-long-timeout\"\n", - " },\n", - " \"spec\": {\n", - " \"annotations\": {\n", - " \"deployment_version\": \"v1\",\n", - "\t \"seldon.io/rest-timeout\":\"100000\",\n", - "\t \"seldon.io/grpc-timeout\":\"100000\"\n", - " },\n", - " \"name\": \"long-to\",\n", - " \"predictors\": [\n", - " {\n", - " \"componentSpecs\": [{\n", - " \"spec\": {\n", - " \"containers\": [\n", - " {\n", - " \"image\": \"seldonio/mock_classifier_grpc:1.5\",\n", - " \"imagePullPolicy\": \"IfNotPresent\",\n", - " \"name\": \"classifier\",\n", - " \"resources\": {\n", - " \"requests\": {\n", - " \"memory\": \"1Mi\"\n", - " }\n", - " }\n", - " }\n", - " ],\n", - " \"terminationGracePeriodSeconds\": 20\n", - " }\n", - " }],\n", - " \"graph\": {\n", - " \"children\": [],\n", - " \"name\": \"classifier\",\n", - " \"endpoint\": {\n", - "\t\t\t\"type\" : \"GRPC\"\n", - "\t\t },\n", - " \"type\": \"MODEL\"\n", - " },\n", - " \"name\": \"test\",\n", - " \"replicas\": 1,\n", - "\t\t\"annotations\": {\n", - "\t\t \"predictor_version\" : \"v1\"\n", - "\t\t}\n", - " }\n", - " ]\n", - " }\n", - "}" + "%%writetemplate resources/model_long_timeouts.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: model-long-timeout\n", + "spec:\n", + " annotations:\n", + " deployment_version: v1\n", + " seldon.io/grpc-timeout: '100000'\n", + " seldon.io/rest-timeout: '100000'\n", + " name: long-to\n", + " predictors:\n", + " - annotations:\n", + " predictor_version: v1\n", + " componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " resources:\n", + " requests:\n", + " memory: 1Mi\n", + " terminationGracePeriodSeconds: 20\n", + " graph:\n", + " children: []\n", + " name: classifier\n", + " type: MODEL\n", + " name: test\n", + " replicas: 1\n" ] }, { @@ -133,22 +164,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/model-long-timeout created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f resources/model_long_timeouts.json -n seldon" + "!kubectl apply -f resources/model_long_timeouts.yaml -n seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"model-long-timeout-test-0-classifier\" successfully rolled out\r\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=model-long-timeout -o jsonpath='{.items[0].metadata.name}')" ] @@ -162,7 +209,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": { "scrolled": true }, @@ -182,11 +229,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 1], 'values': [0.4806932754099743]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'names': ['proba'], 'tensor': {'shape': [1, 1], 'values': [0.08047035772935462]}}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\")\n", "assert(r.success==True)\n", @@ -202,11 +261,23 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "False <_InactiveRpcError of RPC that terminated with:\n", + "\tstatus = StatusCode.RESOURCE_EXHAUSTED\n", + "\tdetails = \"Received message larger than max (8000023 vs. 4194304)\"\n", + "\tdebug_error_string = \"{\"created\":\"@1603887710.710555595\",\"description\":\"Error received from peer ipv6:[::1]:8003\",\"file\":\"src/core/lib/surface/call.cc\",\"file_line\":1061,\"grpc_message\":\"Received message larger than max (8000023 vs. 4194304)\",\"grpc_status\":8}\"\n", + ">\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\",shape=(1000000,1))\n", "print(r.success,r.msg)" @@ -214,11 +285,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"model-long-timeout\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f resources/model_long_timeouts.json" ] @@ -239,85 +318,81 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "scrolled": true - }, + "execution_count": 25, + "metadata": {}, "outputs": [], "source": [ - "%%writefile resources/model_grpc_size.json\n", - "{\n", - " \"apiVersion\": \"machinelearning.seldon.io/v1alpha2\",\n", - " \"kind\": \"SeldonDeployment\",\n", - " \"metadata\": {\n", - " \"labels\": {\n", - " \"app\": \"seldon\"\n", - " },\n", - " \"name\": \"seldon-model\"\n", - " },\n", - " \"spec\": {\n", - " \"annotations\": {\n", - "\t \"seldon.io/grpc-max-message-size\":\"10000000\",\n", - "\t \"seldon.io/rest-timeout\":\"100000\",\n", - "\t \"seldon.io/grpc-timeout\":\"100000\"\n", - " },\n", - " \"name\": \"test-deployment\",\n", - " \"predictors\": [\n", - " {\n", - " \"componentSpecs\": [{\n", - " \"spec\": {\n", - " \"containers\": [\n", - " {\n", - " \"image\": \"seldonio/mock_classifier_grpc:1.5\",\n", - " \"imagePullPolicy\": \"IfNotPresent\",\n", - " \"name\": \"classifier\",\n", - " \"resources\": {\n", - " \"requests\": {\n", - " \"memory\": \"1Mi\"\n", - " }\n", - " }\n", - " }\n", - " ],\n", - " \"terminationGracePeriodSeconds\": 20\n", - " }\n", - " }],\n", - " \"graph\": {\n", - " \"children\": [],\n", - " \"name\": \"classifier\",\n", - " \"endpoint\": {\n", - "\t\t\t\"type\" : \"GRPC\"\n", - "\t\t },\n", - " \"type\": \"MODEL\"\n", - " },\n", - " \"name\": \"grpc-size\",\n", - " \"replicas\": 1,\n", - "\t\t\"annotations\": {\n", - "\t\t \"predictor_version\" : \"v1\"\n", - "\t\t}\n", - " }\n", - " ]\n", - " }\n", - "}" + "%%writetemplate resources/model_grpc_size.yaml\n", + "apiVersion: machinelearning.seldon.io/v1alpha2\n", + "kind: SeldonDeployment\n", + "metadata:\n", + " labels:\n", + " app: seldon\n", + " name: seldon-model\n", + "spec:\n", + " annotations:\n", + " seldon.io/grpc-max-message-size: '10000000'\n", + " seldon.io/grpc-timeout: '100000'\n", + " seldon.io/rest-timeout: '100000'\n", + " name: test-deployment\n", + " predictors:\n", + " - annotations:\n", + " predictor_version: v1\n", + " componentSpecs:\n", + " - spec:\n", + " containers:\n", + " - image: seldonio/mock_classifier:{VERSION}\n", + " imagePullPolicy: IfNotPresent\n", + " name: classifier\n", + " resources:\n", + " requests:\n", + " memory: 1Mi\n", + " terminationGracePeriodSeconds: 20\n", + " graph:\n", + " children: []\n", + " endpoint:\n", + " type: GRPC\n", + " name: classifier\n", + " type: MODEL\n", + " name: grpc-size\n", + " replicas: 1\n" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/seldon-model created\r\n" + ] + } + ], "source": [ - "!kubectl create -f resources/model_grpc_size.json -n seldon" + "!kubectl create -f resources/model_grpc_size.yaml -n seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"seldon-model-grpc-size-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"seldon-model-grpc-size-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=seldon-model -o jsonpath='{.items[0].metadata.name}')" ] @@ -331,11 +406,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "True\n" + ] + } + ], "source": [ "sc = SeldonClient(deployment_name=\"seldon-model\",namespace=\"seldon\",\n", " grpc_max_send_message_length=50 * 1024 * 1024, grpc_max_receive_message_length=50 * 1024 * 1024)\n", @@ -346,11 +429,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": { "scrolled": true }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"seldon-model\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f resources/model_grpc_size.json -n seldon" ] @@ -380,7 +471,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.6.8" }, "varInspector": { "cols": { diff --git a/notebooks/operator_upgrade.ipynb b/notebooks/operator_upgrade.ipynb index 6c232c3c7c..6de3694ffd 100644 --- a/notebooks/operator_upgrade.ipynb +++ b/notebooks/operator_upgrade.ipynb @@ -415,6 +415,13 @@ "source": [ "!kubectl delete sdep --all" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -433,7 +440,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.4" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/notebooks/protocol_examples.ipynb b/notebooks/protocol_examples.ipynb index 29a205413b..eacf1defce 100644 --- a/notebooks/protocol_examples.ipynb +++ b/notebooks/protocol_examples.ipynb @@ -13,6 +13,12 @@ " * grpcurl\n", " * pygmentize\n", " \n", + "## Examples\n", + "\n", + " * [Seldon Protocol](#Seldon-Protocol-Model)\n", + " * [Tensorflow Protocol](#Tensorflow-Protocol-Model)\n", + " * [KFServing V2 Protocol](#KFServing-V2-Protocol-Model)\n", + " \n", "\n", "## Setup Seldon Core\n", "\n", @@ -26,25 +32,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], "source": [ "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -52,183 +74,132 @@ "import time" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Seldon Protocol REST Model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will deploy a REST model that uses the SELDON Protocol namely by specifying the attribute `protocol: seldon`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile resources/model_seldon_rest.yaml\n", - "apiVersion: machinelearning.seldon.io/v1\n", - "kind: SeldonDeployment\n", - "metadata:\n", - " name: rest-seldon\n", - "spec:\n", - " name: restseldon\n", - " protocol: seldon\n", - " transport: rest \n", - " predictors:\n", - " - componentSpecs:\n", - " - spec:\n", - " containers:\n", - " - image: seldonio/mock_classifier_rest:1.3\n", - " name: classifier\n", - " graph:\n", - " name: classifier\n", - " type: MODEL\n", - " name: model\n", - " replicas: 1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f resources/model_seldon_rest.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=rest-seldon -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(60):\n", - " state=!kubectl get sdep rest-seldon -o jsonpath='{.status.state}'\n", - " state=state[0]\n", - " print(state)\n", - " if state==\"Available\":\n", - " break\n", - " time.sleep(1)\n", - "assert(state==\"Available\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now send requests using the Seldon protocol format." - ] - }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ - "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", - " -X POST http://localhost:8003/seldon/seldon/rest-seldon/api/v1.0/predictions \\\n", - " -H \"Content-Type: application/json\"\n", - "d=json.loads(X[0])\n", - "print(d)\n", - "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" + "from IPython.core.magic import register_line_cell_magic\n", + "\n", + "@register_line_cell_magic\n", + "def writetemplate(line, cell):\n", + " with open(line, 'w') as f:\n", + " f.write(cell.format(**globals()))" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'1.5.0-dev'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ - "!kubectl delete -f resources/model_seldon_rest.yaml" + "VERSION=!cat ../version.txt\n", + "VERSION=VERSION[0]\n", + "VERSION" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Seldon Protocol GRPC Model" + "## Seldon Protocol Model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We will deploy a model with a GRPC endpoint that uses the SELDON Protocol namely by specifying the attribute `protocol: seldon`" + "We will deploy a REST model that uses the SELDON Protocol namely by specifying the attribute `protocol: seldon`" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "%%writefile resources/model_seldon_grpc.yaml\n", + "%%writetemplate resources/model_seldon.yaml\n", "apiVersion: machinelearning.seldon.io/v1\n", "kind: SeldonDeployment\n", "metadata:\n", - " name: grpc-seldon\n", + " name: example-seldon\n", "spec:\n", - " name: grpcseldon\n", " protocol: seldon\n", - " transport: grpc\n", " predictors:\n", " - componentSpecs:\n", " - spec:\n", " containers:\n", - " - image: seldonio/mock_classifier_grpc:1.3\n", + " - image: seldonio/mock_classifier:{VERSION}\n", " name: classifier\n", " graph:\n", " name: classifier\n", " type: MODEL\n", - " endpoint:\n", - " type: GRPC\n", " name: model\n", " replicas: 1" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-seldon created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f resources/model_seldon_grpc.yaml" + "!kubectl apply -f resources/model_seldon.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"example-seldon-model-0-classifier\" successfully rolled out\r\n" + ] + } + ], "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=grpc-seldon \\\n", - " -o jsonpath='{.items[0].metadata.name}')" + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-seldon -o jsonpath='{.items[0].metadata.name}')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], "source": [ "for i in range(60):\n", - " state=!kubectl get sdep grpc-seldon -o jsonpath='{.status.state}'\n", + " state=!kubectl get sdep example-seldon -o jsonpath='{.status.state}'\n", " state=state[0]\n", " print(state)\n", " if state==\"Available\":\n", @@ -238,20 +209,43 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 14, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}, 'meta': {}}\n" + ] + } + ], "source": [ - "We can now send a set of GRPC requests using `grpcurl` and leveraging the Seldon protocol" + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/example-seldon/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)\n", + "assert(d[\"data\"][\"ndarray\"][0][0] > 0.4)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'names': ['proba'], 'ndarray': [[0.43782349911420193]]}}\n" + ] + } + ], "source": [ "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[1.0,2.0,5.0]]}}' \\\n", - " -rpc-header seldon:grpc-seldon -rpc-header namespace:seldon \\\n", + " -rpc-header seldon:example-seldon -rpc-header namespace:seldon \\\n", " -plaintext \\\n", " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", "d=json.loads(\"\".join(X))\n", @@ -261,36 +255,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example-seldon\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f resources/model_seldon_grpc.yaml" + "!kubectl delete -f resources/model_seldon.yaml" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Tensorflow Protocol REST Model\n", - "We will deploy a model with a REST endpoint that uses the TENSORLFOW Protocol namely by specifying the attribute `protocol: tensorflow`" + "## Tensorflow Protocol Model\n", + "We will deploy a model that uses the TENSORLFOW Protocol namely by specifying the attribute `protocol: tensorflow`" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting resources/model_tfserving.yaml\n" + ] + } + ], "source": [ - "%%writefile resources/model_tfserving_rest.yaml\n", + "%%writefile resources/model_tfserving.yaml\n", "apiVersion: machinelearning.seldon.io/v1\n", "kind: SeldonDeployment\n", "metadata:\n", - " name: rest-tfserving\n", + " name: example-tfserving\n", "spec:\n", - " name: resttfserving\n", " protocol: tensorflow\n", - " transport: rest\n", " predictors:\n", " - componentSpecs:\n", " - spec:\n", @@ -306,42 +314,71 @@ " - containerPort: 8501\n", " name: http\n", " protocol: TCP\n", + " - containerPort: 8500\n", + " name: grpc\n", + " protocol: TCP\n", " graph:\n", " name: halfplustwo\n", " type: MODEL\n", " endpoint:\n", - " service_port: 8501\n", + " http_port: 8501\n", + " grpc_port: 8500\n", " name: model\n", " replicas: 1" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 18, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/example-tfserving created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f resources/model_tfserving_rest.yaml" + "!kubectl apply -f resources/model_tfserving.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"example-tfserving-model-0-halfplustwo\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"example-tfserving-model-0-halfplustwo\" successfully rolled out\n" + ] + } + ], "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=rest-tfserving \\\n", + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=example-tfserving \\\n", " -o jsonpath='{.items[0].metadata.name}')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], "source": [ "for i in range(60):\n", - " state=!kubectl get sdep rest-tfserving -o jsonpath='{.status.state}'\n", + " state=!kubectl get sdep example-tfserving -o jsonpath='{.status.state}'\n", " state=state[0]\n", " print(state)\n", " if state==\"Available\":\n", @@ -350,21 +387,22 @@ "assert(state==\"Available\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now send requests using the REST tensorflow protocol format." - ] - }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'predictions': [2.5, 3.0, 4.5]}\n" + ] + } + ], "source": [ "X=!curl -s -d '{\"instances\": [1.0, 2.0, 5.0]}' \\\n", - " -X POST http://localhost:8003/seldon/seldon/rest-tfserving/v1/models/halfplustwo/:predict \\\n", + " -X POST http://localhost:8003/seldon/seldon/example-tfserving/v1/models/halfplustwo/:predict \\\n", " -H \"Content-Type: application/json\"\n", "d=json.loads(\"\".join(X))\n", "print(d)\n", @@ -373,112 +411,21 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl delete -f resources/model_tfserving_rest.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Tensorflow Protocol GRPC Model\n", - "We will deploy a model with a GRPC endpoint that uses the TENSOFRLOW Protocol namely by specifying the attribute `protocol: tensorflow`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile resources/model_tfserving_grpc.yaml\n", - "apiVersion: machinelearning.seldon.io/v1\n", - "kind: SeldonDeployment\n", - "metadata:\n", - " name: grpc-tfserving\n", - "spec:\n", - " name: grpctfserving\n", - " protocol: tensorflow\n", - " transport: grpc\n", - " predictors:\n", - " - componentSpecs:\n", - " - spec:\n", - " containers:\n", - " - args: \n", - " - --port=8500\n", - " - --rest_api_port=8501\n", - " - --model_name=halfplustwo\n", - " - --model_base_path=gs://seldon-models/tfserving/half_plus_two\n", - " image: tensorflow/serving\n", - " name: halfplustwo\n", - " ports:\n", - " - containerPort: 8500\n", - " name: grpc\n", - " protocol: TCP\n", - " graph:\n", - " name: halfplustwo\n", - " type: MODEL\n", - " endpoint:\n", - " service_port: 8500\n", - " type: GRPC\n", - " name: model\n", - " replicas: 1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f resources/model_tfserving_grpc.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=grpc-tfserving \\\n", - " -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(60):\n", - " state=!kubectl get sdep grpc-tfserving -o jsonpath='{.status.state}'\n", - " state=state[0]\n", - " print(state)\n", - " if state==\"Available\":\n", - " break\n", - " time.sleep(1)\n", - "assert(state==\"Available\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now send requests using the GRPC tensorflow protocol format." - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 22, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'outputs': {'x': {'dtype': 'DT_FLOAT', 'tensorShape': {'dim': [{'size': '3'}]}, 'floatVal': [2.5, 3, 3.5]}}, 'modelSpec': {'name': 'halfplustwo', 'version': '123', 'signatureName': 'serving_default'}}\n" + ] + } + ], "source": [ "X=!cd ../executor/proto && grpcurl \\\n", " -d '{\"model_spec\":{\"name\":\"halfplustwo\"},\"inputs\":{\"x\":{\"dtype\": 1, \"tensor_shape\": {\"dim\":[{\"size\": 3}]}, \"floatVal\" : [1.0, 2.0, 3.0]}}}' \\\n", - " -rpc-header seldon:grpc-tfserving -rpc-header namespace:seldon \\\n", + " -rpc-header seldon:example-tfserving -rpc-header namespace:seldon \\\n", " -plaintext -proto ./prediction_service.proto \\\n", " 0.0.0.0:8003 tensorflow.serving.PredictionService/Predict\n", "d=json.loads(\"\".join(X))\n", @@ -488,34 +435,50 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"example-tfserving\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f resources/model_tfserving_grpc.yaml" + "!kubectl delete -f resources/model_tfserving.yaml" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## V2 Protocol REST Model" + "## KFServing V2 Protocol Model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "We will deploy a REST model that uses the V2 Protocol namely by specifying the attribute `protocol: kfserving`" + "We will deploy a REST model that uses the KFServing V2 Protocol namely by specifying the attribute `protocol: kfserving`" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting resources/model_v2.yaml\n" + ] + } + ], "source": [ - "%%writefile resources/model_v2_rest.yaml\n", + "%%writefile resources/model_v2.yaml\n", "apiVersion: machinelearning.seldon.io/v1alpha2\n", "kind: SeldonDeployment\n", "metadata:\n", @@ -534,27 +497,52 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 25, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/triton created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f resources/model_v2_rest.yaml" + "!kubectl apply -f resources/model_v2.yaml" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 26, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"triton-simple-0-simple\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"triton-simple-0-simple\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=triton -o jsonpath='{.items[0].metadata.name}')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Available\n" + ] + } + ], "source": [ "for i in range(60):\n", " state=!kubectl get sdep triton -o jsonpath='{.status.state}'\n", @@ -566,18 +554,19 @@ "assert(state==\"Available\")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now send requests using the Seldon protocol format." - ] - }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'model_name': 'simple', 'model_version': '1', 'outputs': [{'name': 'OUTPUT0', 'datatype': 'INT32', 'shape': [1, 16], 'data': [2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32]}, {'name': 'OUTPUT1', 'datatype': 'INT32', 'shape': [1, 16], 'data': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}]}\n" + ] + } + ], "source": [ "X=!curl -s -d '{\"inputs\":[{\"name\":\"INPUT0\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"data\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16],\"datatype\":\"INT32\",\"shape\":[1,16]}]}' \\\n", " -X POST http://0.0.0.0:8003/seldon/seldon/triton/v2/models/simple/infer \\\n", @@ -589,97 +578,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl delete -f resources/model_v2_rest.yaml" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## V2 Protocol GRPC Model" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We will deploy a GRPC model that uses the V2 Protocol namely by specifying the attribute `protocol: kfserving`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%writefile resources/model_v2_grpc.yaml\n", - "apiVersion: machinelearning.seldon.io/v1alpha2\n", - "kind: SeldonDeployment\n", - "metadata:\n", - " name: triton\n", - "spec:\n", - " protocol: kfserving\n", - " transport: grpc\n", - " predictors:\n", - " - graph:\n", - " children: []\n", - " implementation: TRITON_SERVER\n", - " modelUri: gs://seldon-models/trtis/simple-model\n", - " name: simple\n", - " name: simple\n", - " replicas: 1" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl apply -f resources/model_v2_grpc.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=triton -o jsonpath='{.items[0].metadata.name}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for i in range(60):\n", - " state=!kubectl get sdep triton -o jsonpath='{.status.state}'\n", - " state=state[0]\n", - " print(state)\n", - " if state==\"Available\":\n", - " break\n", - " time.sleep(1)\n", - "assert(state==\"Available\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We can now send requests using the Seldon protocol format." - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 29, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{ \"modelName\": \"simple\", \"modelVersion\": \"1\", \"outputs\": [ { \"name\": \"OUTPUT0\", \"datatype\": \"INT32\", \"shape\": [ \"1\", \"16\" ] }, { \"name\": \"OUTPUT1\", \"datatype\": \"INT32\", \"shape\": [ \"1\", \"16\" ] } ], \"rawOutputContents\": [ \"AgAAAAQAAAAGAAAACAAAAAoAAAAMAAAADgAAABAAAAASAAAAFAAAABYAAAAYAAAAGgAAABwAAAAeAAAAIAAAAA==\", \"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==\" ]}\n" + ] + } + ], "source": [ "X=!cd ../executor/api/grpc/kfserving/inference && \\\n", " grpcurl -d '{\"model_name\":\"simple\",\"inputs\":[{\"name\":\"INPUT0\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]},{\"name\":\"INPUT1\",\"contents\":{\"int_contents\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},\"datatype\":\"INT32\",\"shape\":[1,16]}]}' \\\n", @@ -692,11 +601,19 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 30, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"triton\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f resources/model_v2_grpc.yaml" + "!kubectl delete -f resources/model_v2.yaml" ] }, { diff --git a/notebooks/resources/.gitignore b/notebooks/resources/.gitignore index 5f503d2709..fa172cc524 100644 --- a/notebooks/resources/.gitignore +++ b/notebooks/resources/.gitignore @@ -13,3 +13,20 @@ income_explainer.yaml wine_explainer.yaml imagenet_explainer_grpc.yaml mnist_rest_explainer.yaml +model2.yaml +model_seldon.yaml +model_structural_test.yaml +model_tfserving.yaml +model_tracing.yaml +model_v2.yaml +model_v2_mnist.yaml +tracing_config.yaml +model_grpc_size.yaml +model_long_timeouts.yaml +iris-sklearn-v2.yaml +moviesentiment_explainer.yaml +iris.yaml +elasticnet_wine.yaml +halfplustwo_rest.yaml +mnist_rest.yaml +iris-xgboost-v2.yaml diff --git a/notebooks/resources/iris-xgboost-v2.yaml b/notebooks/resources/iris-xgboost-v2.yaml deleted file mode 100644 index 0d86e36da8..0000000000 --- a/notebooks/resources/iris-xgboost-v2.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1 -kind: SeldonDeployment -metadata: - name: xgboost -spec: - name: iris - protocol: kfserving - predictors: - - graph: - children: [] - implementation: XGBOOST_SERVER - modelUri: gs://seldon-models/xgboost/iris - name: classifier - name: default - replicas: 1 diff --git a/notebooks/resources/model_long_timeouts.json b/notebooks/resources/model_long_timeouts.json index 702f8e9615..f0d18c3c37 100644 --- a/notebooks/resources/model_long_timeouts.json +++ b/notebooks/resources/model_long_timeouts.json @@ -20,7 +20,7 @@ "spec": { "containers": [ { - "image": "seldonio/mock_classifier_grpc:1.5", + "image": "seldonio/mock_classifier:1.5.0-dev", "imagePullPolicy": "IfNotPresent", "name": "classifier", "resources": { @@ -36,9 +36,6 @@ "graph": { "children": [], "name": "classifier", - "endpoint": { - "type" : "GRPC" - }, "type": "MODEL" }, "name": "test", diff --git a/notebooks/resources/model_seldon_rest.yaml b/notebooks/resources/model_seldon_rest.yaml index c2b4f4cc4c..9c22f43d9a 100644 --- a/notebooks/resources/model_seldon_rest.yaml +++ b/notebooks/resources/model_seldon_rest.yaml @@ -10,7 +10,7 @@ spec: - componentSpecs: - spec: containers: - - image: seldonio/mock_classifier_rest:1.3 + - image: seldonio/mock_classifier:1.3.0-dev name: classifier graph: name: classifier diff --git a/notebooks/resources/model_tfserving_rest.yaml b/notebooks/resources/model_tfserving_rest.yaml index 35ade433f0..637d5fddd3 100644 --- a/notebooks/resources/model_tfserving_rest.yaml +++ b/notebooks/resources/model_tfserving_rest.yaml @@ -1,11 +1,9 @@ apiVersion: machinelearning.seldon.io/v1 kind: SeldonDeployment metadata: - name: rest-tfserving + name: example-tfserving spec: - name: resttfserving protocol: tensorflow - transport: rest predictors: - componentSpecs: - spec: @@ -20,10 +18,15 @@ spec: ports: - containerPort: 8501 name: http + protocol: TCP + - containerPort: 8500 + name: grpc + protocol: TCP graph: name: halfplustwo type: MODEL endpoint: - service_port: 8501 + http_port: 8501 + grpc_port: 8500 name: model replicas: 1 diff --git a/notebooks/resources/moviesentiment_explainer.yaml b/notebooks/resources/moviesentiment_explainer.yaml index 8ec55436c2..30edeb04e9 100644 --- a/notebooks/resources/moviesentiment_explainer.yaml +++ b/notebooks/resources/moviesentiment_explainer.yaml @@ -15,4 +15,4 @@ spec: explainer: type: AnchorText name: default - replicas: 1 + replicas: 1 \ No newline at end of file diff --git a/notebooks/server_examples.ipynb b/notebooks/server_examples.ipynb index 7034feb8ba..e50a87950f 100644 --- a/notebooks/server_examples.ipynb +++ b/notebooks/server_examples.ipynb @@ -13,20 +13,45 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error from server (AlreadyExists): namespaces \"seldon\" already exists\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"kind-kind\" modified.\r\n" + ] + } + ], + "source": [ + "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" + "import json" ] }, { @@ -55,9 +80,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting ../servers/sklearnserver/samples/iris.yaml\n" + ] + } + ], "source": [ "%%writefile ../servers/sklearnserver/samples/iris.yaml\n", "apiVersion: machinelearning.seldon.io/v1alpha2\n", @@ -89,18 +122,34 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/sklearn created\r\n" + ] + } + ], "source": [ "!kubectl apply -f ../servers/sklearnserver/samples/iris.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "deployment \"sklearn-default-0-classifier\" successfully rolled out\r\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=sklearn -o jsonpath='{.items[0].metadata.name}')" ] @@ -112,9 +161,37 @@ "Once it's deployed we can send our sklearn model requests" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### REST Requests" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': ['t:0', 't:1', 't:2'], 'ndarray': [[9.912315378486718e-07, 0.0007015931307743852, 0.9992974156376878]]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0, 6.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/sklearn/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -124,9 +201,33 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 4\n", + " values: 0.6806334106409295\n", + " values: 0.28830360482888695\n", + " values: 0.8975654688578534\n", + " values: 0.24493223527153596\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['t:0', 't:1', 't:2'], 'tensor': {'shape': [1, 3], 'values': [0.14449927217770223, 0.5104835080299303, 0.34501721979236755]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\",shape=(1,4))\n", "print(r)\n", @@ -137,14 +238,76 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "And delete the model we deployed" + "#### gRPC Requests" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 4], 'values': [0.22157381246204433, 0.020827790786912548, 0.7633031884959527, 0.19217598516001178]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'names': ['t:0', 't:1', 't:2'], 'tensor': {'shape': [1, 3], 'values': [0.10758420949891327, 0.4796849750770796, 0.4127308154240072]}}}\n" + ] + } + ], + "source": [ + "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\",shape=(1,4))\n", + "print(r)\n", + "assert(r.success==True)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'names': ['t:0', 't:1', 't:2'], 'ndarray': [[9.912315378486718e-07, 0.0007015931307743852, 0.9992974156376878]]}}\n" + ] + } + ], + "source": [ + "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[1.0,2.0,5.0,6.0]]}}' \\\n", + " -rpc-header seldon:sklearn -rpc-header namespace:seldon \\\n", + " -plaintext \\\n", + " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)" + ] + }, + { + "cell_type": "markdown", "metadata": {}, - "outputs": [], + "source": [ + "And delete the model we deployed" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"sklearn\" deleted\r\n" + ] + } + ], "source": [ "!kubectl delete -f ../servers/sklearnserver/samples/iris.yaml" ] @@ -161,9 +324,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting ./resources/iris-sklearn-v2.yaml\n" + ] + } + ], "source": [ "%%writefile ./resources/iris-sklearn-v2.yaml\n", "apiVersion: machinelearning.seldon.io/v1\n", @@ -192,18 +363,35 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/sklearn created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f ../testing/resources/iris-sklearn-v2.yaml" + "!kubectl apply -f resources/iris-sklearn-v2.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"sklearn-default-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=sklearn -o jsonpath='{.items[0].metadata.name}')" ] @@ -218,9 +406,35 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"model_name\": \"classifier\",\n", + " \"model_version\": \"v1\",\n", + " \"id\": \"5a0bce13-98e8-4588-9fc5-7c8b11140174\",\n", + " \"parameters\": null,\n", + " \"outputs\": [\n", + " {\n", + " \"name\": \"predict\",\n", + " \"shape\": [\n", + " 1\n", + " ],\n", + " \"datatype\": \"FP32\",\n", + " \"parameters\": null,\n", + " \"data\": [\n", + " 2\n", + " ]\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], "source": [ "import requests\n", "import json\n", @@ -252,11 +466,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"sklearn\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f ../testing/resources/iris-sklearn-v2.yaml" + "!kubectl delete -f resources/iris-sklearn-v2.yaml" ] }, { @@ -285,11 +507,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing resources/iris.yaml\n" + ] + } + ], "source": [ - "%%writefile ../servers/xgboostserver/samples/iris.yaml\n", + "%%writefile resources/iris.yaml\n", "apiVersion: machinelearning.seldon.io/v1\n", "kind: SeldonDeployment\n", "metadata:\n", @@ -315,18 +545,35 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/xgboost created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f ../servers/xgboostserver/samples/iris.yaml" + "!kubectl apply -f resources/iris.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"xgboost-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"xgboost-default-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=xgboost -o jsonpath='{.items[0].metadata.name}')" ] @@ -335,12 +582,33 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Once it's deployed we can send our xgboost model requests" + "#### Rest Requests" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': [], 'ndarray': [2.0]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[1.0, 2.0, 5.0, 6.0]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/xgboost/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -350,9 +618,33 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 4\n", + " values: 0.617330874549778\n", + " values: 0.4955768380888388\n", + " values: 0.8325074250157055\n", + " values: 0.29623988728724804\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': [], 'tensor': {'shape': [1], 'values': [0.0]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\",shape=(1,4))\n", "print(r)\n", @@ -363,16 +655,78 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "And delete the model we deployed" + "#### gRPC Requests" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 4], 'values': [0.7029137333313852, 0.9422923262038959, 0.7271094013077166, 0.18443703566724434]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1], 'values': [0.0]}}}\n" + ] + } + ], + "source": [ + "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\",shape=(1,4))\n", + "print(r)\n", + "assert(r.success==True)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'ndarray': [2]}}\n" + ] + } + ], + "source": [ + "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[1.0,2.0,5.0,6.0]]}}' \\\n", + " -rpc-header seldon:xgboost -rpc-header namespace:seldon \\\n", + " -plaintext \\\n", + " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)" + ] + }, + { + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "!kubectl delete -f ../servers/xgboostserver/samples/iris.yaml" + "And delete the model we deployed" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"xgboost\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f resources/iris.yaml" ] }, { @@ -387,9 +741,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting ./resources/iris-xgboost-v2.yaml\n" + ] + } + ], "source": [ "%%writefile ./resources/iris-xgboost-v2.yaml\n", "apiVersion: machinelearning.seldon.io/v1\n", @@ -418,18 +780,35 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/xgboost created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f ../testing/resources/iris-xgboost-v2.yaml" + "!kubectl apply -f ./resources/iris-xgboost-v2.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"xgboost-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"xgboost-default-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=xgboost -o jsonpath='{.items[0].metadata.name}')" ] @@ -444,9 +823,35 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"model_name\": \"classifier\",\n", + " \"model_version\": \"v1\",\n", + " \"id\": \"d1887e45-e8c3-4644-b72b-5a3148463917\",\n", + " \"parameters\": null,\n", + " \"outputs\": [\n", + " {\n", + " \"name\": \"predict\",\n", + " \"shape\": [\n", + " 1\n", + " ],\n", + " \"datatype\": \"FP32\",\n", + " \"parameters\": null,\n", + " \"data\": [\n", + " 2.0\n", + " ]\n", + " }\n", + " ]\n", + "}\n" + ] + } + ], "source": [ "import requests\n", "import json\n", @@ -478,11 +883,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"xgboost\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f ../testing/resources/iris-xgboost-v2.yaml" + "!kubectl delete -f ./resources/iris-xgboost-v2.yaml" ] }, { @@ -511,11 +924,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting ./resources/mnist_rest.yaml\n" + ] + } + ], "source": [ - "%%writefile ../servers/tfserving/samples/mnist_rest.yaml\n", + "%%writefile ./resources/mnist_rest.yaml\n", "apiVersion: machinelearning.seldon.io/v1alpha2\n", "kind: SeldonDeployment\n", "metadata:\n", @@ -535,59 +956,909 @@ " - name: model_name\n", " type: STRING\n", " value: mnist-model\n", + " - name: model_input\n", + " type: STRING\n", + " value: images\n", + " - name: model_output\n", + " type: STRING\n", + " value: scores \n", " name: default\n", " replicas: 1" ] }, { - "cell_type": "markdown", - "metadata": {}, + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/tfserving created\r\n" + ] + } + ], "source": [ - "Once it's deployed we can send our sklearn model requests" + "!kubectl apply -f ./resources/mnist_rest.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"tfserving-default-0-mnist-model\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"tfserving-default-0-mnist-model\" successfully rolled out\n" + ] + } + ], "source": [ - "!kubectl apply -f ../servers/tfserving/samples/mnist_rest.yaml" + "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tfserving -o jsonpath='{.items[0].metadata.name}')" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ - "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=tfserving -o jsonpath='{.items[0].metadata.name}')" + "from seldon_core.seldon_client import SeldonClient\n", + "sc = SeldonClient(deployment_name=\"tfserving\",namespace=\"seldon\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Once it's deployed we can send our sklearn model requests" + "#### REST Request" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 784\n", + " values: 0.5360208678294703\n", + " values: 0.5063574371425368\n", + " values: 0.9443243610713166\n", + " values: 0.0011084051211286416\n", + " values: 0.20099891756759192\n", + " values: 0.30071217406011774\n", + " values: 0.15622672261211623\n", + " values: 0.032474907686516064\n", + " values: 0.7440398045429419\n", + " values: 0.7108256047143693\n", + " values: 0.7645229494651484\n", + " values: 0.457444409411902\n", + " values: 0.316314894432161\n", + " values: 0.7350234986133874\n", + " values: 0.6847025510931343\n", + " values: 0.3124896535821692\n", + " values: 0.5023732148295806\n", + " values: 0.7766729114301516\n", + " values: 0.31117856330829097\n", + " values: 0.145499075089073\n", + " values: 0.9955898498586168\n", + " values: 0.04865057935951689\n", + " values: 0.2557675256095512\n", + " values: 0.3312323613365653\n", + " values: 0.5179019306893689\n", + " values: 0.2909692479700965\n", + " values: 0.6482350619102406\n", + " values: 0.23776475362315974\n", + " values: 0.9684044714121269\n", + " values: 0.3946100097962236\n", + " values: 0.6735207815070934\n", + " values: 0.4898909692131146\n", + " values: 0.7554495042153699\n", + " values: 0.9323323200773203\n", + " values: 0.8252476428579795\n", + " values: 0.22463738461912197\n", + " values: 0.8516178728688074\n", + " values: 0.828740934139463\n", + " values: 0.923362928072387\n", + " values: 0.2714093376678782\n", + " values: 0.9054254676779065\n", + " values: 0.13043825073432547\n", + " values: 0.6652904285620423\n", + " values: 0.6868063268085522\n", + " values: 0.5238440260046521\n", + " values: 0.6220355519313966\n", + " values: 0.94549067747297\n", + " values: 0.37350607942680747\n", + " values: 0.5940961179246904\n", + " values: 0.9875444499853954\n", + " values: 0.8441678775402586\n", + " values: 0.11603782952161557\n", + " values: 0.37060645993619024\n", + " values: 0.5223202975793629\n", + " values: 0.811955792147166\n", + " values: 0.3435665722918425\n", + " values: 0.6008945947691681\n", + " values: 0.7369476957371978\n", + " values: 0.42091588878275954\n", + " values: 0.7657478466544629\n", + " values: 0.7701005025913801\n", + " values: 0.7374120251216275\n", + " values: 0.040805007022627904\n", + " values: 0.8584078709275651\n", + " values: 0.4423009671290504\n", + " values: 0.463903678216087\n", + " values: 0.512994097178021\n", + " values: 0.21951215945925662\n", + " values: 0.39961265703661286\n", + " values: 0.3740776385162834\n", + " values: 0.8533328271293943\n", + " values: 0.6189058705460576\n", + " values: 0.7065177099136564\n", + " values: 0.7242754580877176\n", + " values: 0.4849158210968818\n", + " values: 0.7814732453001029\n", + " values: 0.8578162974062509\n", + " values: 0.14444494780372197\n", + " values: 0.20601734865077448\n", + " values: 0.31146170009259744\n", + " values: 0.02548904815923836\n", + " values: 0.03206555155558377\n", + " values: 0.06961913601940051\n", + " values: 0.5836905987975214\n", + " values: 0.6164841160182376\n", + " values: 0.18932462791453508\n", + " values: 0.7640015155876169\n", + " values: 0.5070826654549139\n", + " values: 0.09628169062880776\n", + " values: 0.19382869540563252\n", + " values: 0.9412062339456596\n", + " values: 0.5966486866762393\n", + " values: 0.6206143916246194\n", + " values: 0.19628479152988265\n", + " values: 0.1707854491649463\n", + " values: 0.8825761502582269\n", + " values: 0.9808711531596559\n", + " values: 0.08009655223214474\n", + " values: 0.1701636661745568\n", + " values: 0.11406709402446225\n", + " values: 0.053396793006944\n", + " values: 0.09360205809605349\n", + " values: 0.4293152516177\n", + " values: 0.8096761824364951\n", + " values: 0.45331815671809395\n", + " values: 0.0028687829212518112\n", + " values: 0.5383002126233998\n", + " values: 0.8374588688774007\n", + " values: 0.9065778173249387\n", + " values: 0.6928498685521003\n", + " values: 0.8287977470411556\n", + " values: 0.24420517062946456\n", + " values: 0.7087713010840551\n", + " values: 0.5597533577586793\n", + " values: 0.8340435101340461\n", + " values: 0.2834350433264451\n", + " values: 0.43389553731337505\n", + " values: 0.10779407741501923\n", + " values: 0.6810241672979269\n", + " values: 0.23028774938902108\n", + " values: 0.5338486723691872\n", + " values: 0.4476058035980687\n", + " values: 0.9769456277119264\n", + " values: 0.769714449533957\n", + " values: 0.8542344400369551\n", + " values: 0.3955913030396866\n", + " values: 0.6048931767815328\n", + " values: 0.3950530370957115\n", + " values: 0.3097862997602735\n", + " values: 0.6932151492719949\n", + " values: 0.045388147129425405\n", + " values: 0.3547049416354636\n", + " values: 0.2841549434007644\n", + " values: 0.5662212925051878\n", + " values: 0.11093317271366854\n", + " values: 0.705020697930456\n", + " values: 0.7701571220581236\n", + " values: 0.37401247234766277\n", + " values: 0.1649242055093013\n", + " values: 0.9522789591359156\n", + " values: 0.8298189406272996\n", + " values: 0.508548667739561\n", + " values: 0.5964131984844762\n", + " values: 0.6662052512807753\n", + " values: 0.18822217394822516\n", + " values: 0.1014283639141419\n", + " values: 0.2727041959497004\n", + " values: 0.13195607979659518\n", + " values: 0.654358242653656\n", + " values: 0.010816820290051932\n", + " values: 0.3246004560914473\n", + " values: 0.18662535569944738\n", + " values: 0.9969794986052404\n", + " values: 0.8179132942771283\n", + " values: 0.41450015472170887\n", + " values: 0.7057910168766867\n", + " values: 0.2899871260679674\n", + " values: 0.3831315267402442\n", + " values: 0.6964699607754848\n", + " values: 0.6902136692430094\n", + " values: 0.06504699984184215\n", + " values: 0.2212401546585464\n", + " values: 0.5098525158853898\n", + " values: 0.6542913471191273\n", + " values: 0.5305648126211812\n", + " values: 0.5261761585451143\n", + " values: 0.08846718053044222\n", + " values: 0.5449922096065108\n", + " values: 0.450353963044702\n", + " values: 0.736257854300933\n", + " values: 0.8772451907530697\n", + " values: 0.27841102672226525\n", + " values: 0.3033740515060188\n", + " values: 0.6171396233233576\n", + " values: 0.6951649288619974\n", + " values: 0.8182446576595515\n", + " values: 0.253567955939659\n", + " values: 0.12296965314136521\n", + " values: 0.6876519958686744\n", + " values: 0.5084038381279407\n", + " values: 0.3162575542640982\n", + " values: 0.4756606086109816\n", + " values: 0.43825582762402704\n", + " values: 0.6520559128506422\n", + " values: 0.2158293236314267\n", + " values: 0.31308462396962444\n", + " values: 0.6440740562434655\n", + " values: 0.4009300295558108\n", + " values: 0.6978264213104906\n", + " values: 0.6616370709891428\n", + " values: 0.6367887648607818\n", + " values: 0.31540480702480955\n", + " values: 0.3193423487030481\n", + " values: 0.6365777067910425\n", + " values: 0.31303332464825295\n", + " values: 0.6435042020660738\n", + " values: 0.08208548622076128\n", + " values: 0.7319588884879634\n", + " values: 0.13465892702216542\n", + " values: 0.9251573225160918\n", + " values: 0.345573001143858\n", + " values: 0.9559961076064103\n", + " values: 0.8823235599425719\n", + " values: 0.733970597324987\n", + " values: 0.5696271754971727\n", + " values: 0.8350173497452276\n", + " values: 0.9302611112741331\n", + " values: 0.085730512951118\n", + " values: 0.3260059956477296\n", + " values: 0.9599402637813182\n", + " values: 0.17161133880228752\n", + " values: 0.5602601381442434\n", + " values: 0.6723815023775268\n", + " values: 0.7078626873859801\n", + " values: 0.7496493694017511\n", + " values: 0.5677968849211822\n", + " values: 0.9603337925981938\n", + " values: 0.25382929789896536\n", + " values: 0.4385898470949069\n", + " values: 0.3138678435894573\n", + " values: 0.014716958785869316\n", + " values: 0.8328494171577923\n", + " values: 0.5258691205199649\n", + " values: 0.9834458233457017\n", + " values: 0.8299393874252645\n", + " values: 0.9479573125486154\n", + " values: 0.5243863173442517\n", + " values: 0.6362091138797941\n", + " values: 0.5331937898727952\n", + " values: 0.11332070887964063\n", + " values: 0.7487479278612228\n", + " values: 0.9928224639029607\n", + " values: 0.8502747017784754\n", + " values: 0.0321492211390938\n", + " values: 0.34307485934605775\n", + " values: 0.8787030807910386\n", + " values: 0.9319853617653254\n", + " values: 0.48388167639980806\n", + " values: 0.7705939507093756\n", + " values: 0.41580560999962324\n", + " values: 0.7163837582563678\n", + " values: 0.9146320183508799\n", + " values: 0.6971199681031134\n", + " values: 0.5539744623288142\n", + " values: 0.25995074498950166\n", + " values: 0.9494048850887903\n", + " values: 0.9000981143723509\n", + " values: 0.008773296862421343\n", + " values: 0.7096514746184078\n", + " values: 0.02064151322069885\n", + " values: 0.5775845783029481\n", + " values: 0.19856229453824958\n", + " values: 0.24536839099750452\n", + " values: 0.3795205620190526\n", + " values: 0.4566006281824123\n", + " values: 0.27488251204081726\n", + " values: 0.5230736920721154\n", + " values: 0.944786925908848\n", + " values: 0.40578249769908437\n", + " values: 0.7030616018485285\n", + " values: 0.1840656119435634\n", + " values: 0.30121888994181156\n", + " values: 0.19815642221456198\n", + " values: 0.49897954392454935\n", + " values: 0.5367149427022465\n", + " values: 0.9138835741389495\n", + " values: 0.5667413688454215\n", + " values: 0.6953082201603188\n", + " values: 0.5707191680455059\n", + " values: 0.112579217486654\n", + " values: 0.7935170856924955\n", + " values: 0.04482086979683375\n", + " values: 0.13114584183372735\n", + " values: 0.5591425594559639\n", + " values: 0.419359076001929\n", + " values: 0.2902845471556895\n", + " values: 0.7933085568467363\n", + " values: 0.19502077327703005\n", + " values: 0.7252628320221692\n", + " values: 0.49568307297056735\n", + " values: 0.026721004773544332\n", + " values: 0.2012609186244132\n", + " values: 0.21566624700097836\n", + " values: 0.11319887296239373\n", + " values: 0.05322925310391857\n", + " values: 0.9061548739680476\n", + " values: 0.5489280987735055\n", + " values: 0.542982362073632\n", + " values: 0.08366846681967754\n", + " values: 0.2572692705243168\n", + " values: 0.49416904551079566\n", + " values: 0.32230111011811136\n", + " values: 0.7591235832328593\n", + " values: 0.22041583463594616\n", + " values: 0.9170683383323374\n", + " values: 0.5344754610086917\n", + " values: 0.564590851997434\n", + " values: 0.12815992080600713\n", + " values: 0.8719111483012745\n", + " values: 0.035840622603644734\n", + " values: 0.6760823684252939\n", + " values: 0.8501341063860512\n", + " values: 0.7341254275366301\n", + " values: 0.9090459841266889\n", + " values: 0.07242515674522643\n", + " values: 0.9536426693629991\n", + " values: 0.7350260813438712\n", + " values: 0.6288298247965987\n", + " values: 0.7931915805484405\n", + " values: 0.17024771017234186\n", + " values: 0.953252686301089\n", + " values: 0.7540189984426959\n", + " values: 0.7532987512070413\n", + " values: 0.9929276810306974\n", + " values: 0.5713769353402813\n", + " values: 0.8249831085323256\n", + " values: 0.08069521883648301\n", + " values: 0.5695828449612735\n", + " values: 0.3620309940389933\n", + " values: 0.932151964546219\n", + " values: 0.14000218826010258\n", + " values: 0.2565483179525061\n", + " values: 0.12472709333907672\n", + " values: 0.8783578053336435\n", + " values: 0.8498249761122714\n", + " values: 0.43682317822049754\n", + " values: 0.17018463173500065\n", + " values: 0.3926843519548764\n", + " values: 0.302441440408318\n", + " values: 0.7363589688121549\n", + " values: 0.45900581547840913\n", + " values: 0.2236886879049732\n", + " values: 0.6273578319298534\n", + " values: 0.6294505669649296\n", + " values: 0.49730292070432236\n", + " values: 0.009527472486263622\n", + " values: 0.27833314596730263\n", + " values: 0.7149960197384322\n", + " values: 0.9732186795495434\n", + " values: 0.4448929993535524\n", + " values: 0.538517439034856\n", + " values: 0.08173441497780032\n", + " values: 0.9864220830735781\n", + " values: 0.273003929918351\n", + " values: 0.09591969149079482\n", + " values: 0.850005546063552\n", + " values: 0.35179177338337897\n", + " values: 0.8640085165206244\n", + " values: 0.5233229508243511\n", + " values: 0.9003830249424318\n", + " values: 0.9183543156670848\n", + " values: 0.42028752080260257\n", + " values: 0.668557111849264\n", + " values: 0.9993036693900328\n", + " values: 0.5789869826427709\n", + " values: 0.778659399146549\n", + " values: 0.1552856832480649\n", + " values: 0.403152439838838\n", + " values: 0.9706977970836598\n", + " values: 0.17918312695083116\n", + " values: 0.7622577911849607\n", + " values: 0.7919950174725843\n", + " values: 0.5857861816580721\n", + " values: 0.25197663376295276\n", + " values: 0.5412053051748058\n", + " values: 0.09858777854079204\n", + " values: 0.6150346708058114\n", + " values: 0.9328979768920286\n", + " values: 0.9044363189250925\n", + " values: 0.559176995740771\n", + " values: 0.1494730291537517\n", + " values: 0.6888998931163487\n", + " values: 0.2975138608576282\n", + " values: 0.9389664151771026\n", + " values: 0.8285814027545962\n", + " values: 0.8464201527440463\n", + " values: 0.04546503717293271\n", + " values: 0.37853901600330364\n", + " values: 0.2797125426563424\n", + " values: 0.3209557534350721\n", + " values: 0.01226286876314131\n", + " values: 0.5134745626594431\n", + " values: 0.34564719406384126\n", + " values: 0.5481530262116232\n", + " values: 0.276811016700345\n", + " values: 0.4534978552924782\n", + " values: 0.6255281920097239\n", + " values: 0.2413858038446035\n", + " values: 0.3076962791723832\n", + " values: 0.40603124182377315\n", + " values: 0.3086621982490061\n", + " values: 0.1437331233994702\n", + " values: 0.1255991648072735\n", + " values: 0.9271479436232434\n", + " values: 0.32338121280380927\n", + " values: 0.5806125138393727\n", + " values: 0.4466621712821769\n", + " values: 0.7966711518075326\n", + " values: 0.27877216289455853\n", + " values: 0.7932192259052537\n", + " values: 0.16739908835122053\n", + " values: 0.3896623336313255\n", + " values: 0.44152757454411873\n", + " values: 0.9609342343030184\n", + " values: 0.4300305574891332\n", + " values: 0.03772930218876691\n", + " values: 0.10900720423108778\n", + " values: 0.1574385720583351\n", + " values: 0.5116762708239543\n", + " values: 0.3998389439060407\n", + " values: 0.03338422468319402\n", + " values: 0.3990138778909599\n", + " values: 0.210985165243171\n", + " values: 0.7063235960084482\n", + " values: 0.35818005531503994\n", + " values: 0.19875977937493017\n", + " values: 0.09954845660572298\n", + " values: 0.969049376647686\n", + " values: 0.8404252755939952\n", + " values: 0.8701935122100497\n", + " values: 0.12698995075507324\n", + " values: 0.3687442759603681\n", + " values: 0.9347781458087433\n", + " values: 0.7667640427586535\n", + " values: 0.07629811846914081\n", + " values: 0.20970740776529218\n", + " values: 0.2685408167480068\n", + " values: 0.5076886141183455\n", + " values: 0.6011676038953925\n", + " values: 0.12091280816746264\n", + " values: 0.8091541559771682\n", + " values: 0.8995615527810743\n", + " values: 0.06929060965454181\n", + " values: 0.17753071167227785\n", + " values: 0.8133260883044017\n", + " values: 0.8589720175195685\n", + " values: 0.9336864659890087\n", + " values: 0.6768455889044129\n", + " values: 0.2638655191001119\n", + " values: 0.9507694756291761\n", + " values: 0.24690415797640852\n", + " values: 0.7832166085385622\n", + " values: 0.8731473079215657\n", + " values: 0.29236725593755364\n", + " values: 0.6765269724417613\n", + " values: 0.8169304642588561\n", + " values: 0.2526842624685981\n", + " values: 0.2994039873765125\n", + " values: 0.4197211357689281\n", + " values: 0.411687236443233\n", + " values: 0.05944284012558332\n", + " values: 0.7603469257491514\n", + " values: 0.37863125453254187\n", + " values: 0.1430736568345673\n", + " values: 0.9509137166407023\n", + " values: 0.6789831094676481\n", + " values: 0.2543115547281979\n", + " values: 0.7285781373868861\n", + " values: 0.8224295468229845\n", + " values: 0.11807445663216187\n", + " values: 0.8015447555088723\n", + " values: 0.2573733836316786\n", + " values: 0.24585026623080364\n", + " values: 0.24076836622223896\n", + " values: 0.9729463624245652\n", + " values: 0.4548457530914721\n", + " values: 0.4706016801697369\n", + " values: 0.12923173456874426\n", + " values: 0.5503704712987648\n", + " values: 0.25388420395153743\n", + " values: 0.7580523814652274\n", + " values: 0.13126194925037526\n", + " values: 0.4037522209819333\n", + " values: 0.3714636961466423\n", + " values: 0.9346183399968756\n", + " values: 0.41742734070362886\n", + " values: 0.15409673026375892\n", + " values: 0.6021763830491289\n", + " values: 0.846672082658356\n", + " values: 0.12423825248053977\n", + " values: 0.6868477069420822\n", + " values: 0.0759550512481525\n", + " values: 0.5067646660361566\n", + " values: 0.7856327708459713\n", + " values: 0.3631801570285813\n", + " values: 0.25703694531748844\n", + " values: 0.12943974691818727\n", + " values: 0.8714725804319144\n", + " values: 0.421640733731833\n", + " values: 0.026442101268956297\n", + " values: 0.13616254036264086\n", + " values: 0.4032943006647307\n", + " values: 0.8377019859641086\n", + " values: 0.5797136730312334\n", + " values: 0.46748579413587255\n", + " values: 0.047817981296037315\n", + " values: 0.3742264697609722\n", + " values: 0.9260596735871129\n", + " values: 0.9168240583346488\n", + " values: 0.026712191380598105\n", + " values: 0.858602547884268\n", + " values: 0.4482814102365975\n", + " values: 0.491917511718446\n", + " values: 0.29463091802129937\n", + " values: 0.8358501503690211\n", + " values: 0.8503235456924544\n", + " values: 0.40391688613790955\n", + " values: 0.0701343040305088\n", + " values: 0.4745823520941944\n", + " values: 0.9029320859449049\n", + " values: 0.4503531083546918\n", + " values: 0.24635242652824607\n", + " values: 0.9379058309791022\n", + " values: 0.3987017084261504\n", + " values: 0.12215298971897803\n", + " values: 0.40341739296019163\n", + " values: 0.4967405901301486\n", + " values: 0.7610197742613506\n", + " values: 0.7360065464236473\n", + " values: 0.251974801937636\n", + " values: 0.9671917696069435\n", + " values: 0.10456468432260813\n", + " values: 0.6614612311553544\n", + " values: 0.8687510717037961\n", + " values: 0.05868205353576994\n", + " values: 0.5329304736613242\n", + " values: 0.2863842640000569\n", + " values: 0.7180643725465106\n", + " values: 0.12754145991850885\n", + " values: 0.2897552735151431\n", + " values: 0.10304121398593247\n", + " values: 0.09318368358363638\n", + " values: 0.36525961573507093\n", + " values: 0.003011313936758575\n", + " values: 0.7596195244911024\n", + " values: 0.5451693156647526\n", + " values: 0.6358854779051621\n", + " values: 0.2725727860091439\n", + " values: 0.12988115942365364\n", + " values: 0.4392426474375739\n", + " values: 0.8605539045993147\n", + " values: 0.13609102664816464\n", + " values: 0.751828337067231\n", + " values: 0.3344234856497711\n", + " values: 0.17651451228995296\n", + " values: 0.4560547813327971\n", + " values: 0.8411605235157206\n", + " values: 0.5385277879408291\n", + " values: 0.2711168710176214\n", + " values: 0.6912197035883472\n", + " values: 0.9346325117337705\n", + " values: 0.7808561250712892\n", + " values: 0.6335569708239671\n", + " values: 0.26499619542150754\n", + " values: 0.20949712253438735\n", + " values: 0.2684525540688789\n", + " values: 0.35390862682236324\n", + " values: 0.6463373911790108\n", + " values: 0.7866178467130873\n", + " values: 0.08516742519947895\n", + " values: 0.2241413818967387\n", + " values: 0.14430747570541047\n", + " values: 0.5611145233899633\n", + " values: 0.31271661151347874\n", + " values: 0.5990164605536633\n", + " values: 0.9092589659087726\n", + " values: 0.30880076800872247\n", + " values: 0.9532213078962992\n", + " values: 0.7835929639488887\n", + " values: 0.4399492468694832\n", + " values: 0.13015117862073378\n", + " values: 0.9872450750763808\n", + " values: 0.9955715196049365\n", + " values: 0.111619561700122\n", + " values: 0.8601859617989108\n", + " values: 0.12256895905002796\n", + " values: 0.3768730160527528\n", + " values: 0.8325151286888005\n", + " values: 0.2963666837965908\n", + " values: 0.11105890943463692\n", + " values: 0.4174293055087349\n", + " values: 0.4783949994336353\n", + " values: 0.20665456715907438\n", + " values: 0.03774481978849786\n", + " values: 0.8996315532454533\n", + " values: 0.9297944483662742\n", + " values: 0.393784257827529\n", + " values: 0.6678005089923453\n", + " values: 0.04011207645732329\n", + " values: 0.509632230242221\n", + " values: 0.2569122760130199\n", + " values: 0.40342736206781404\n", + " values: 0.8246619667884595\n", + " values: 0.8496489919433845\n", + " values: 0.08582446297128599\n", + " values: 0.33939438015737333\n", + " values: 0.09319400646900222\n", + " values: 0.02663915870064215\n", + " values: 0.9788174759544884\n", + " values: 0.6097687931300745\n", + " values: 0.9532361105070625\n", + " values: 0.0015798867987968368\n", + " values: 0.8882840572040731\n", + " values: 0.5322430312923632\n", + " values: 0.1738081658937669\n", + " values: 0.40878404844834026\n", + " values: 0.774837385057174\n", + " values: 0.6483242913820617\n", + " values: 0.53374185513516\n", + " values: 0.11059486089591986\n", + " values: 0.34912679754213105\n", + " values: 0.28183821905862516\n", + " values: 0.23333213467323555\n", + " values: 0.7112757818053341\n", + " values: 0.238558069642419\n", + " values: 0.3706436886737656\n", + " values: 0.06523055282784507\n", + " values: 0.21961186071143068\n", + " values: 0.5923917111823842\n", + " values: 0.439367831611718\n", + " values: 0.32517125920671475\n", + " values: 0.7377119867571327\n", + " values: 0.3743469783612958\n", + " values: 0.6050554776678365\n", + " values: 0.5635606797546017\n", + " values: 0.703353620401727\n", + " values: 0.8139659306699663\n", + " values: 0.061567129859247616\n", + " values: 0.3105298484121516\n", + " values: 0.8298542068206634\n", + " values: 0.3096386293383021\n", + " values: 0.2689431716091151\n", + " values: 0.28842728430722975\n", + " values: 0.21124434864873542\n", + " values: 0.14708622835540763\n", + " values: 0.48644381443092954\n", + " values: 0.19684129634435676\n", + " values: 0.08270150814922528\n", + " values: 0.9516970403641595\n", + " values: 0.950825780064431\n", + " values: 0.21581181385533732\n", + " values: 0.7788293836347402\n", + " values: 0.026129742574298787\n", + " values: 0.7146848568512404\n", + " values: 0.31655713763416593\n", + " values: 0.3400160812659132\n", + " values: 0.24619366736061887\n", + " values: 0.9345567992529578\n", + " values: 0.7207843263800611\n", + " values: 0.4802938852560793\n", + " values: 0.5882251478244057\n", + " values: 0.597376370492188\n", + " values: 0.047389556086804396\n", + " values: 0.2691252825103355\n", + " values: 0.06039961422913909\n", + " values: 0.6561738024461861\n", + " values: 0.6217700407519451\n", + " values: 0.7611077408132957\n", + " values: 0.9602815302513494\n", + " values: 0.1822575667892845\n", + " values: 0.5664221020248803\n", + " values: 0.8366560443511608\n", + " values: 0.7448274487999165\n", + " values: 0.4414716514983362\n", + " values: 0.37008913215800576\n", + " values: 0.5223847001257356\n", + " values: 0.039718144281731704\n", + " values: 0.07003672560834573\n", + " values: 0.42547166795304014\n", + " values: 0.4435013803058856\n", + " values: 0.1624962637269236\n", + " values: 0.6853128550753754\n", + " values: 0.7557098279363763\n", + " values: 0.7589232058107265\n", + " values: 0.8686713841601104\n", + " values: 0.41887751952933205\n", + " values: 0.5024352752601032\n", + " values: 0.7409763245812273\n", + " values: 0.1790214520585407\n", + " values: 0.4867550326305259\n", + " values: 0.5528404260521127\n", + " values: 0.03979410619482093\n", + " values: 0.5090184134734697\n", + " values: 0.2504015119608278\n", + " values: 0.6684089819713986\n", + " values: 0.6604954416316245\n", + " values: 0.15705298937359147\n", + " values: 0.3318305009338641\n", + " values: 0.9722486604898151\n", + " values: 0.5005235821058707\n", + " values: 0.7982253235671071\n", + " values: 0.5490609051552953\n", + " values: 0.47287251000745145\n", + " values: 0.613227252187827\n", + " values: 0.7920239276743556\n", + " values: 0.03954996384701914\n", + " values: 0.7666887466321749\n", + " values: 0.735092086378898\n", + " values: 0.5312634811871918\n", + " values: 0.43230376548852034\n", + " values: 0.5312358403012936\n", + " values: 0.6770071555479892\n", + " values: 0.345503304869635\n", + " values: 0.37078434252413395\n", + " values: 0.3377350535534914\n", + " values: 0.3500488657191112\n", + " values: 0.9698692426520431\n", + " values: 0.7736231723042446\n", + " values: 0.10789461429300018\n", + " values: 0.38082233459342196\n", + " values: 0.984838687638348\n", + " values: 0.9683082164201839\n", + " values: 0.6569659568769912\n", + " values: 0.674349634286518\n", + " values: 0.09560396731679122\n", + " values: 0.854815490288653\n", + " values: 0.7084915013066769\n", + " values: 0.8986435021083526\n", + " values: 0.7358256453909476\n", + " values: 0.46491719664706355\n", + " values: 0.7515300853592601\n", + " values: 0.38370141817820325\n", + " values: 0.16001364360598058\n", + " values: 0.7694804594191055\n", + " values: 0.6796671925567934\n", + " values: 0.22712144277232027\n", + " values: 0.4437395244106861\n", + " values: 0.296328720399331\n", + " values: 0.6190527024484993\n", + " values: 0.4332062790598099\n", + " values: 0.8939576498027775\n", + " values: 0.5593274338206903\n", + " values: 0.4653369936054328\n", + " values: 0.042741042529305884\n", + " values: 0.3771371715828278\n", + " values: 0.396307160231545\n", + " values: 0.5490910317081658\n", + " values: 0.7164393199005844\n", + " values: 0.7101149804135912\n", + " values: 0.3340930466059656\n", + " values: 0.451592835540773\n", + " values: 0.20314419135672113\n", + " values: 0.4199307446114321\n", + " values: 0.4516435543118411\n", + " values: 0.31852359896664506\n", + " values: 0.18951603364347958\n", + " values: 0.886769898060098\n", + " values: 0.6188339287024962\n", + " values: 0.16144570806476022\n", + " values: 0.16345201174416824\n", + " values: 0.1742034834311299\n", + " values: 0.5567121794180873\n", + " values: 0.8965890704589345\n", + " values: 0.34283945755516976\n", + " values: 0.9394261281044933\n", + " values: 0.20235446302342863\n", + " values: 0.28679783387001956\n", + " values: 0.52862959458994\n", + " values: 0.038847159278550536\n", + " values: 0.6142707527677586\n", + " values: 0.15042934463288282\n", + " values: 0.5777053657975023\n", + " values: 0.7054292993326041\n", + " values: 0.9091213599169915\n", + " values: 0.6524798644836082\n", + " values: 0.8012867563617284\n", + " values: 0.9611337407318353\n", + " values: 0.06253041973517492\n", + " values: 0.273785686273178\n", + " values: 0.5997565342975181\n", + " values: 0.5758718298752356\n", + " values: 0.960330073752839\n", + " values: 0.7904602624316882\n", + " values: 0.9142698108991233\n", + " values: 0.04248413170435006\n", + " values: 0.8914884341643144\n", + " values: 0.7228223588980498\n", + " values: 0.3837908997748416\n", + " values: 0.48018490613682197\n", + " values: 0.026673821506872253\n", + " values: 0.8956279671167092\n", + " values: 0.8758779463188948\n", + " values: 0.20139213249231736\n", + " values: 0.5127573005832172\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': ['t:0', 't:1', 't:2', 't:3', 't:4', 't:5', 't:6', 't:7', 't:8', 't:9'], 'tensor': {'shape': [1, 10], 'values': [1.48347365e-17, 2.75936967e-34, 0.966874599, 0.0310300216, 2.9558217e-28, 0.00209478312, 1.02384538e-13, 1.48587807e-13, 5.29912654e-07, 1.38094071e-18]}}, 'meta': {}}\n" + ] + } + ], + "source": [ + "r = sc.predict(gateway=\"ambassador\",transport=\"rest\",shape=(1,784))\n", + "print(r)\n", + "assert(r.success==True)" + ] + }, + { + "cell_type": "markdown", "metadata": {}, - "outputs": [], "source": [ - "from seldon_core.seldon_client import SeldonClient\n", - "sc = SeldonClient(deployment_name=\"tfserving\",namespace=\"seldon\")" + "#### gRPC Request" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 784], 'values': [0.47209083184647593, 0.8838373150382969, 0.9181145850457837, 0.24859620806264326, 0.23293720880908908, 0.4941481171884624, 0.10725857979635278, 0.5287047511828012, 0.19695290280618072, 0.10247878859133952, 0.9738834180569024, 0.21298312177597456, 0.7460646984755017, 0.7849562914857755, 0.5704135117601201, 0.02792477113467806, 0.7793940787278147, 0.606568611346314, 0.29803598789024754, 0.1326626326173992, 0.32915350266835375, 0.0883570944110672, 0.7607531788790823, 0.1623233904716811, 0.5821722444997973, 0.6970934469577975, 0.35932488456489753, 0.06432456300653788, 0.9265409211003779, 0.38557298397196615, 0.9963670945752263, 0.37694931788140484, 0.16640281034611637, 0.9045008098130327, 0.305694315056507, 0.7073745635364587, 0.23511637628623283, 0.4853187768831849, 0.8487579535826453, 0.9742161341598627, 0.2840885206281021, 0.9946856297748927, 0.20052979354792833, 0.7052308229990519, 0.7816445781358587, 0.9641751912908683, 0.8009730609813319, 0.20525311831057635, 0.17804631307550012, 0.32294296644305454, 0.22608903077216602, 0.6684456585413396, 0.17319867096341646, 0.1359225577596581, 0.19271881055181728, 0.9007234700346303, 0.6036351882523099, 0.37590296271040147, 0.22239192537896624, 0.4902561773551054, 0.5152205571873875, 0.27018092125292825, 0.5428585446231049, 0.9505027222941741, 0.31436977354273665, 0.5363975522028566, 0.6973076187350311, 0.5385679200411694, 0.3046181780524051, 0.9795945243622183, 0.6688625475855863, 0.27797121684342874, 0.7698168684612438, 0.6405064896531694, 0.741136641010486, 0.4422756346986155, 0.5020037498161358, 0.45267571835522624, 0.6987439551430717, 0.07023790256343765, 0.06538735896120074, 0.8447674465772355, 0.6834811377368004, 0.026655828948662252, 0.628407066460694, 0.7569493959847295, 0.5277510788613616, 0.9515171094715688, 0.0422695830276012, 0.7843265416837605, 0.46317391631367033, 0.583661958205498, 0.3644196723211566, 0.26096235874249907, 0.3948508616615424, 0.0023061899186325174, 0.45307204390142086, 0.8516423964629364, 0.03707642073067863, 0.9271640937902405, 0.409206302378464, 0.661599798590934, 0.027983004135873735, 0.588005676007795, 0.2828618373713707, 0.7877912809062873, 0.343145658281636, 0.07377717295125297, 0.6071652489063395, 0.6007041775493313, 0.23223753649321188, 0.7612948390663158, 0.44447266321239565, 0.9273598123195708, 0.282978441929136, 0.2128351693570546, 0.1429002965300663, 0.7439925843340691, 0.7167155918625596, 0.724251714444177, 0.8725660162835415, 0.7424029115069352, 0.9916136416330926, 0.8482468061429691, 0.7436136762568578, 0.3679970021888629, 0.5046901155711218, 0.45718301949304396, 0.8602511895942554, 0.26069535790710174, 0.6957808747450142, 0.12463938278567599, 0.49036698498767084, 0.9166202047688227, 0.9589055767319941, 0.5197770404503094, 0.10754965761372348, 0.30627239870310063, 0.9939793276058694, 0.6872347042768607, 0.598788096267154, 0.5565468005777939, 0.08563594120260676, 0.7519502681021503, 0.644847542639883, 0.027710472980757905, 0.14069045123233948, 0.7810300575219142, 0.4795574487611761, 0.30901153601434916, 0.9581244787569426, 0.03214852586373984, 0.29481851385874014, 0.797765241930748, 0.19345404846072112, 0.5286088784173252, 0.5883122340074208, 0.04195058450608091, 0.06812170861295319, 0.15536689368634182, 0.7908763541239359, 0.07913898022548893, 0.1405428597409839, 0.7178766992095842, 0.2201986029432006, 0.3444018253089439, 0.6590924926281457, 0.027922953628436775, 0.6229219392541856, 0.7942818701382509, 0.08641977471547879, 0.4292054462303935, 0.7971997103423022, 0.7799101458336517, 0.4134929471507769, 0.28154573972838737, 0.8501706069059192, 0.06684091159580718, 0.40661794048439004, 0.1218763457740969, 0.9645344066460255, 0.05521652568124946, 0.7343069369873275, 0.5658627483379838, 0.4678796722196298, 0.863310136820279, 0.5238952155905634, 0.5065908081612522, 0.8294390661468148, 0.37451807408930027, 0.48477800757953404, 0.6953102725753888, 0.3405941116011548, 0.9030593326629215, 0.627887410950056, 0.16175630415580455, 0.2243601258247344, 0.3867521046378264, 0.22379942072598824, 0.4672441342611249, 0.32023905668140473, 0.4281673649020987, 0.035325685079042546, 0.15052927926842696, 0.3261764676491624, 0.8754562200858411, 0.740298727265006, 0.8493116011785045, 0.8463625535751811, 0.9362019934714408, 0.6405404218633977, 0.20887994136306554, 0.5507118599414694, 0.42999328670310766, 0.878822876952522, 0.9237888711710195, 0.21240648069290158, 0.9519785205771032, 0.02488214966365554, 0.03137960908790016, 0.7735585549553864, 0.8816583369145903, 0.0343463916468264, 0.30414554854446596, 0.003463422658347759, 0.928870680878914, 0.4717206133485732, 0.492056523484678, 0.8177266349964164, 0.3687465976380361, 0.3365510245139989, 0.6791977410609507, 0.6023662735170212, 0.35347157840834176, 0.9927939329493394, 0.20929105648466795, 0.302123303331166, 0.4914889551685664, 0.5400037187498605, 0.9830929767399018, 0.22714530615050754, 0.5789590661583232, 0.6897981142287198, 0.10889691271550894, 0.4384196300591371, 0.5228435516331674, 0.4969421216198292, 0.8020903072873768, 0.46635386929247746, 0.7696579472858122, 0.23824577875570807, 0.35125022203987966, 0.6359415865736422, 0.3485456913357483, 0.7336629849891037, 0.3838575696675358, 0.6505575293374803, 0.9275884825202618, 0.7306400550700054, 0.3058299729225683, 0.2668252081663548, 0.09898795146480266, 0.9378669095406443, 0.6264911589377825, 0.0803887329166918, 0.871824757436611, 0.3115981971110533, 0.3552258703702187, 0.12219804034887849, 0.0730451300088304, 0.9501148063114484, 0.11421157046467201, 0.236780210006315, 0.3333738056400689, 0.408002673850983, 0.19708856276958064, 0.4035926356056353, 0.9108541896638964, 0.328496423547042, 0.7591337908099229, 0.040319480766025095, 0.4138100395090991, 0.2718520074516634, 0.5078152310846948, 0.9045921508298108, 0.2167820508550541, 0.33347100044084166, 0.5691623649172347, 0.059086512190570484, 0.8243671033660146, 0.07204175512754307, 0.12000607619533055, 0.5286832752621596, 0.8610537903075214, 0.8481749440406554, 0.6973637822533374, 0.6475937624876916, 0.4085898069490911, 0.7992534786573317, 0.20882979595089513, 0.34581762307733177, 0.5003232822347654, 0.3843766061038987, 0.8411500761248792, 0.18018689081677175, 0.5253280752693897, 0.5724702778732659, 0.9806680534270276, 0.3847449420524539, 0.9430258811514864, 0.8603329386940549, 0.7929652560414097, 0.266828373449958, 0.03952941375940333, 0.6549656093830053, 0.14152709113787532, 0.787167929129303, 0.259619280311448, 0.4533151745029852, 0.454118004461032, 0.8345408782817395, 0.8221526863643673, 0.3481590895146328, 0.5367162452404466, 0.4294252467999541, 0.3889998554079842, 0.3418332127406666, 0.5709287224375224, 0.17035806778661966, 0.6764058068098342, 0.44872194319684133, 0.7950615571819444, 0.4646533693305245, 0.9725239680094071, 0.005476824262743318, 0.19914431292621704, 0.4925242682269947, 0.48260992546534165, 0.2909949297110176, 0.6566314968415025, 0.44440408686108934, 0.3967527076640256, 0.5101739312203654, 0.6768335643056183, 0.18581916702243795, 0.44935273789152186, 0.13863215244214333, 0.7380405681194989, 0.863159014140691, 0.7934827490975779, 0.6247756031028584, 0.5286525682943479, 0.4447449074956633, 0.2624615617293766, 0.5804750089816443, 0.5032150461794586, 0.2929708813668864, 0.739082405614764, 0.5257210368061717, 0.04581074257934703, 0.552344127982606, 0.973586868366023, 0.36545859799316516, 0.370268863583197, 0.8876758840952345, 0.7913075686633906, 0.8317001946992272, 0.986314295068825, 0.8834812342998419, 0.316017224999796, 0.7277287069082552, 0.3420587315387572, 0.054810008370773766, 0.3815081063219917, 0.16672760047661872, 0.7760106435710138, 0.9935306111577131, 0.19517100091548711, 0.2868603953529657, 0.5743120318167182, 0.7901441139022825, 0.5157002827851968, 0.43361141475413434, 0.04533569350178357, 0.00908661666738253, 0.3178585490523962, 0.2686980125322316, 0.07423104248835954, 0.3279763742202987, 0.9601626795685374, 0.752670971985685, 0.43883230417832, 0.9579146263336628, 0.07939534717623775, 0.06594083665462158, 0.9685016794028203, 0.8653914995337819, 0.5991712614990325, 0.6892340494270582, 0.1941142722853184, 0.9863768196189449, 0.2705681966933374, 0.7744808190449916, 0.1399563285915656, 0.212412624788058, 0.98474633716238, 0.6796894522020704, 0.9872514875238475, 0.921756150184594, 0.6583964448611601, 0.9605781392491913, 0.2709662432358748, 0.32265219915294274, 0.7658751429377155, 0.872963927439459, 0.41740803900503276, 0.199142773064663, 0.9431075794580247, 0.7456597313362159, 0.8020654452549147, 0.12175916436925183, 0.6052463634204198, 0.48748291217729256, 0.6514744546160767, 0.9943668103729445, 0.06562262082862658, 0.05985862081952009, 0.5412436299424798, 0.19110446460812447, 0.928732316553108, 0.8426617355500626, 0.9358134713441078, 0.747663047686213, 0.8702098190212978, 0.9145815358251143, 0.9623111255921901, 0.556191330369937, 0.5350011038388929, 0.890621150696471, 0.8527520705572819, 0.8010988183368555, 0.7236127713773604, 0.47054721890933726, 0.7587369070033196, 0.5107816139808784, 0.4190674412194352, 0.21784018602324484, 0.44017796025360756, 0.7872618242426632, 0.20794907993486256, 0.9916174667571579, 0.9051880668359634, 0.9318957509917497, 0.12647160035260907, 0.18141186639132834, 0.48650416851409706, 0.2894128000876053, 0.29722710544030173, 0.5456459387795368, 0.8704900448288221, 0.12638773513156676, 0.20600578910526168, 0.21465930941187839, 0.23415059978453479, 0.4491197828099335, 0.9488254533290725, 0.2398170749597831, 0.06169514929465225, 0.7097961939357247, 0.9131333468327416, 0.2247982882691204, 0.18085980533832313, 0.6742144643351851, 0.7393985087526693, 0.21075831841962778, 0.4103042820619752, 0.5787298531829382, 0.9210908371084175, 0.16987392107321508, 0.10616995448509925, 0.9401242382685034, 0.5583922752182234, 0.8512953150296644, 0.2850720850183087, 0.5699407464417716, 0.4847977039421052, 0.9999004876102908, 0.7503777935611753, 0.6276940933714206, 0.05775246886565255, 0.2061187779307565, 0.8806765197889713, 0.8357234606936236, 0.8323059225482385, 0.9117119199161854, 0.49528636540220816, 0.5911604280420939, 0.17704028461894838, 0.3408194156046882, 0.5225949000059471, 0.5362220451842812, 0.526713261399165, 0.2545149027349616, 0.05539012865974735, 0.7587760815150464, 0.3023564316837012, 0.24277990302764485, 0.6962196654100498, 0.8714104753129002, 0.046796667499896194, 0.1778938097405629, 0.4520325850141438, 0.62163655482298, 0.34740914105934817, 0.7617603551540251, 0.38619097891649334, 0.07063659858805016, 0.29426329792823736, 0.8003076024166746, 0.6433975932470543, 0.5081624965007524, 0.4780812449022942, 0.2534272428340729, 0.5897953038862579, 0.9137940947104325, 0.20151455810577334, 0.774686031272745, 0.4033970915903179, 0.3920828568544067, 0.36604031717644026, 0.854630601552637, 0.4340196373604053, 0.6823003154088585, 0.7406035154438139, 0.38602533429167485, 0.17498923179336956, 0.03615909274010065, 0.545774542186746, 0.7314767326563041, 0.34660725303834294, 0.06737982385278662, 0.9663717207036022, 0.9126674052779989, 0.614318540821783, 0.9030793373833906, 0.006774701170043085, 0.3039757609453946, 0.08661087098902953, 0.36708084090586857, 0.8527955038132641, 0.29272922240428856, 0.9315010543427967, 0.7603074953854699, 0.0802440245051762, 0.804556035822707, 0.12267222660158572, 0.013704541489903965, 0.6184138426201897, 0.4936428452152388, 0.2796373259782361, 0.16221820329693304, 0.06471396767111204, 0.37411062348111757, 0.3683644473804464, 0.789498982112438, 0.5329042139794582, 0.371874026122785, 0.46248842322494543, 0.8528788057278847, 0.5137063454739463, 0.002880682484862729, 0.6026710639391195, 0.14122397375377105, 0.6638464713457064, 0.5519026452007485, 0.5680532990876237, 0.17772571394449344, 0.7282227395003659, 0.9742711112240345, 0.4391460561696493, 0.35982030395367814, 0.8306029123532283, 0.7651408216248665, 0.43829866405872775, 0.18321170537315812, 0.7150178140695751, 0.04967403908115409, 0.8770917664871549, 0.2442421001453452, 0.9448888515682335, 0.5017848067826373, 0.5786406259169061, 0.6339141345026164, 0.21622418217897932, 0.7585553034343241, 0.7594212811877421, 0.045296642330073134, 0.7236382120343704, 0.30377687106618745, 0.22263316779007947, 0.8267810936061025, 0.47605027832286384, 0.4942255148628497, 0.27782216517813485, 0.3915023197075903, 0.3451778751043656, 0.651591544418787, 0.2867266203578416, 0.5806507596218992, 0.06829690606298144, 0.8275549501248533, 0.4297662965824457, 0.42949854036793766, 0.8636185763520072, 0.41067083827001905, 0.3800757646296382, 0.14691996704533594, 0.674385199096699, 0.09125225915877777, 0.9740388495259313, 0.20529471568173674, 0.7084783202103114, 0.4395144292221105, 0.5622739086204741, 0.972973647870242, 0.8525168791852975, 0.28904650749857186, 0.7609456388078955, 0.2897710368195845, 0.8829515370002321, 0.7822996730486097, 0.7675537462595611, 0.6359266835947512, 0.7317268327880497, 0.49727960546730743, 0.19236916397247716, 0.0002702198960089497, 0.6238910904344986, 0.48350657854443313, 0.22571574673330608, 0.20790641396673515, 0.5640847224538025, 0.2558227549847921, 0.682218862802099, 0.7933319859472042, 0.8176632984334289, 0.8590344581637116, 0.6051886263065774, 0.33484859749340246, 0.7253407201724044, 0.42317687803511317, 0.34201609216513595, 0.3222628683413714, 0.15376665348998786, 0.7888501628064104, 0.6008189290434037, 0.4276812295254413, 0.9877971397828393, 0.8362650287984462, 0.9479745429961567, 0.12289541589821673, 0.6593181582485729, 0.7134528227614213, 0.3073027128479674, 0.8240949294110582, 0.8960613735624486, 0.164951359539809, 0.6924044230384172, 0.30422364197216933, 0.8327068565273399, 0.9682620873453123, 0.39393221021793845, 0.6207768674562115, 0.7791602081346949, 0.6216425826880323, 0.5439309269012221, 0.399998883320941, 0.07008036444511045, 0.515655828376216, 0.2651820532086483, 0.2964690910631955, 0.8447312496717595, 0.267121657531443, 0.6757253807687433, 0.11990988163601923, 0.4791330059470148, 0.6857602019275719, 0.9136114244868613, 0.752855793519951, 0.747355630404103, 0.9276317915603618, 0.11642204972437564, 0.12692966796061833, 0.8514924247719169, 0.6972453659310377, 0.8697480762965908, 0.9314845579127051, 0.5728648636049015, 0.47161829258450216, 0.6794977015997933, 0.4929959386227105, 0.40595497583144424, 0.5741549508839735, 0.6412266003150837, 0.9918475532667549, 0.7451254570769575, 0.8726792789562513, 0.7123901649894392, 0.6304908656434686, 0.8192107775801503, 0.49867700538826343, 0.6367937285610142, 0.6087872825506192, 0.5430037804062361, 0.019482531480218612, 0.7762026748139628, 0.28181259893928734, 0.31825670710315224, 0.3482541692130835, 0.03889069034238479, 0.8148277051166162, 0.6045201338979536, 0.1687491467110962, 0.7840527404628816, 0.4341346125000545, 0.0744630334251869, 0.2614636970141073, 0.222806257020164, 0.6816907608473006, 0.6185998964064066, 0.9868774332729169, 0.822558657243309, 0.39241556810124756, 0.79763427342774, 0.6884445863469614, 0.09068776855280614, 0.7926436663695322, 0.6888672287163125, 0.31804983906196094, 0.877099090705907, 0.3907487340360998, 0.34157038742159185, 0.6260515147824817, 0.259730159689165, 0.61490894679174, 0.8465254470950556, 0.25421113043379595, 0.44142208684190887, 0.6133519644397994, 0.7991549321881197, 0.9599874938132772, 0.9623559384867231, 0.03620324039653888, 0.3194553087206853, 0.22650975576328458, 0.48711261632804015, 0.3580418522906287, 0.4329273397882162, 0.9822700676187989, 0.1617745352021036, 0.08226001301979025, 0.9130352351843993, 0.8641031818608453, 0.8435256842312339, 0.4124937198331623, 0.37819116101082906, 0.4484359302463572, 0.824376797366782, 0.27002473137847705, 0.8351297333417245, 0.10014708403086359, 0.4973306861301242, 0.2800750161748923, 0.5926455132915689, 0.6648935288637702, 0.33363023024041993, 0.15108029183468608, 0.534828082621992, 0.8596342876640728, 0.6997591340903839, 0.9665523282670481, 0.16438456502272758, 0.7958694234133213, 0.6635013395295192]}}}\n", + "Response:\n", + "{'data': {'tftensor': {'dtype': 'DT_FLOAT', 'tensorShape': {'dim': [{'size': '1'}, {'size': '10'}]}, 'floatVal': [3.673101e-19, 8.99173e-31, 0.010555523, 0.7877868, 1.0459939e-33, 0.20165771, 2.191654e-14, 1.8198068e-21, 2.565744e-08, 3.1694407e-21]}}}\n" + ] + } + ], "source": [ - "r = sc.predict(gateway=\"ambassador\",transport=\"rest\",shape=(1,784))\n", + "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\",shape=(1,784))\n", "print(r)\n", "assert(r.success==True)" ] @@ -601,11 +1872,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"tfserving\" deleted\r\n" + ] + } + ], "source": [ - "!kubectl delete -f ../servers/tfserving/samples/mnist_rest.yaml" + "!kubectl delete -f ./resources/mnist_rest.yaml" ] }, { @@ -620,11 +1899,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting ./resources/halfplustwo_rest.yaml\n" + ] + } + ], "source": [ - "%%writefile ../servers/tfserving/samples/halfplustwo_rest.yaml\n", + "%%writefile ./resources/halfplustwo_rest.yaml\n", "apiVersion: machinelearning.seldon.io/v1alpha2\n", "kind: SeldonDeployment\n", "metadata:\n", @@ -649,27 +1936,52 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/hpt created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f ../servers/tfserving/samples/halfplustwo_rest.yaml" + "!kubectl apply -f ./resources/halfplustwo_rest.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"hpt-default-0-halfplustwo\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"hpt-default-0-halfplustwo\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=hpt -o jsonpath='{.items[0].metadata.name}')" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'predictions': [2.5, 3.0, 4.5]}\n" + ] + } + ], "source": [ "import json\n", "X=!curl -s -d '{\"instances\": [1.0, 2.0, 5.0]}' \\\n", @@ -682,11 +1994,43 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'outputs': {'x': {'dtype': 'DT_FLOAT', 'tensorShape': {'dim': [{'size': '3'}]}, 'floatVal': [2.5, 3, 3.5]}}, 'modelSpec': {'name': 'halfplustwo', 'version': '123', 'signatureName': 'serving_default'}}\n" + ] + } + ], "source": [ - "!kubectl delete -f ../servers/tfserving/samples/halfplustwo_rest.yaml" + "X=!cd ../executor/proto && grpcurl \\\n", + " -d '{\"model_spec\":{\"name\":\"halfplustwo\"},\"inputs\":{\"x\":{\"dtype\": 1, \"tensor_shape\": {\"dim\":[{\"size\": 3}]}, \"floatVal\" : [1.0, 2.0, 3.0]}}}' \\\n", + " -rpc-header seldon:hpt -rpc-header namespace:seldon \\\n", + " -plaintext -proto ./prediction_service.proto \\\n", + " 0.0.0.0:8003 tensorflow.serving.PredictionService/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)\n", + "assert(d[\"outputs\"][\"x\"][\"floatVal\"][0] == 2.5)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"hpt\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f ./resources/halfplustwo_rest.yaml" ] }, { @@ -700,11 +2044,19 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Overwriting ./resources/elasticnet_wine.yaml\n" + ] + } + ], "source": [ - "%%writefile ../servers/mlflowserver/samples/elasticnet_wine.yaml\n", + "%%writefile ./resources/elasticnet_wine.yaml\n", "apiVersion: machinelearning.seldon.io/v1alpha2\n", "kind: SeldonDeployment\n", "metadata:\n", @@ -747,25 +2099,70 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io/mlflow created\r\n" + ] + } + ], "source": [ - "!kubectl apply -f ../servers/mlflowserver/samples/elasticnet_wine.yaml" + "!kubectl apply -f ./resources/elasticnet_wine.yaml" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Waiting for deployment \"mlflow-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\n", + "deployment \"mlflow-default-0-classifier\" successfully rolled out\n" + ] + } + ], "source": [ "!kubectl rollout status deploy/$(kubectl get deploy -l seldon-deployment-id=mlflow -o jsonpath='{.items[0].metadata.name}')" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### REST requests" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'data': {'names': [], 'ndarray': [5.275558760255382]}, 'meta': {}}\n" + ] + } + ], + "source": [ + "X=!curl -s -d '{\"data\": {\"ndarray\":[[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.1]]}}' \\\n", + " -X POST http://localhost:8003/seldon/seldon/mlflow/api/v1.0/predictions \\\n", + " -H \"Content-Type: application/json\"\n", + "d=json.loads(X[0])\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -775,23 +2172,121 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "meta {\n", + "}\n", + "data {\n", + " tensor {\n", + " shape: 1\n", + " shape: 11\n", + " values: 0.358817295772466\n", + " values: 0.42774982123748906\n", + " values: 0.7525229974243451\n", + " values: 0.4837952054509831\n", + " values: 0.03971051054852981\n", + " values: 0.7977411786519535\n", + " values: 0.843901872160243\n", + " values: 0.5876114963145197\n", + " values: 0.6716037188345559\n", + " values: 0.8544908465852457\n", + " values: 0.3759460042213313\n", + " }\n", + "}\n", + "\n", + "Response:\n", + "{'data': {'names': [], 'tensor': {'shape': [1], 'values': [5.2069715754680725]}}, 'meta': {}}\n" + ] + } + ], "source": [ "r = sc.predict(gateway=\"ambassador\",transport=\"rest\",shape=(1,11))\n", "print(r)\n", "assert(r.success==True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### gRPC Requests" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'meta': {}, 'data': {'ndarray': [5.275558760255382]}}\n" + ] + } + ], + "source": [ + "X=!cd ../executor/proto && grpcurl -d '{\"data\":{\"ndarray\":[[0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1.0,1.1]]}}' \\\n", + " -rpc-header seldon:mlflow -rpc-header namespace:seldon \\\n", + " -plaintext \\\n", + " -proto ./prediction.proto 0.0.0.0:8003 seldon.protos.Seldon/Predict\n", + "d=json.loads(\"\".join(X))\n", + "print(d)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Success:True message:\n", + "Request:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1, 11], 'values': [0.35040541408334647, 0.9104661327491711, 0.4362897953086141, 0.07252654389819702, 0.5452850097930405, 0.8161583853649494, 0.9842675249693481, 0.6805591646597872, 0.6430042482083499, 0.7862682374046126, 0.5803183012135262]}}}\n", + "Response:\n", + "{'meta': {}, 'data': {'tensor': {'shape': [1], 'values': [5.226165754389033]}}}\n" + ] + } + ], + "source": [ + "r = sc.predict(gateway=\"ambassador\",transport=\"grpc\",shape=(1,11))\n", + "print(r)\n", + "assert(r.success==True)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "seldondeployment.machinelearning.seldon.io \"mlflow\" deleted\r\n" + ] + } + ], + "source": [ + "!kubectl delete -f ./resources/elasticnet_wine.yaml" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], - "source": [ - "!kubectl delete -f ../servers/mlflowserver/samples/elasticnet_wine.yaml" - ] + "source": [] } ], "metadata": { @@ -810,7 +2305,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.8" + "version": "3.6.8" }, "varInspector": { "cols": { diff --git a/operator/Makefile b/operator/Makefile index ac6e83639e..4cc0d5cb82 100644 --- a/operator/Makefile +++ b/operator/Makefile @@ -173,8 +173,7 @@ redhat-image-scan: docker push scan.connect.redhat.com/ospid-7f50cebe-122b-495a-a143-41426dfcb6c9/${IMG_VERSION_REDHAT} kind-image-install: docker-build - docker save ${IMG} > operator.tar - kind load image-archive operator.tar --name ${KIND_NAME} + kind load -v 3 docker-image ${IMG} --name ${KIND_NAME} kind-image-install-redhat: docker-build-redhat docker save ${IMG_REDHAT} > operator.tar diff --git a/operator/apis/machinelearning.seldon.io/v1/prepack.go b/operator/apis/machinelearning.seldon.io/v1/prepack.go index 71def9e8b8..aeec4055b2 100644 --- a/operator/apis/machinelearning.seldon.io/v1/prepack.go +++ b/operator/apis/machinelearning.seldon.io/v1/prepack.go @@ -10,20 +10,16 @@ import ( ) const ( - EnvSklearnServerImageRestRelated = "RELATED_IMAGE_SKLEARNSERVER_REST" - EnvSklearnserverImageGrpcRelated = "RELATED_IMAGE_SKLEARNSERVER_GRPC" - EnvXgboostserverImageRestRelated = "RELATED_IMAGE_XGBOOSTSERVER_REST" - EnvXgboostserverImageGrpcRelated = "RELATED_IMAGE_XGBOOSTSERVER_GRPC" - EnvMlflowserverImageRestRelated = "RELATED_IMAGE_MLFLOWSERVER_REST" - EnvMlflowserverImageGrpcRelated = "RELATED_IMAGE_MLFLOWSERVER_GRPC" - EnvTensorflowImageRelated = "RELATED_IMAGE_TENSORFLOW" - EnvTfproxyImageRestRelated = "RELATED_IMAGE_TFPROXY_REST" - EnvTfproxyImageGrpcRelated = "RELATED_IMAGE_TFPROXY_GRPC" - PrepackTensorflowName = "TENSORFLOW_SERVER" - PrepackSklearnName = "SKLEARN_SERVER" - PrepackXgboostName = "XGBOOST_SERVER" - PrepackMlflowName = "MLFLOW_SERVER" - PrepackTritonName = "TRITON_SERVER" + EnvSklearnServerImageRelated = "RELATED_IMAGE_SKLEARNSERVER" + EnvXgboostserverImageRelated = "RELATED_IMAGE_XGBOOSTSERVER" + EnvMlflowserverImageRelated = "RELATED_IMAGE_MLFLOWSERVER" + EnvTensorflowImageRelated = "RELATED_IMAGE_TENSORFLOW" + EnvTfproxyImageRelated = "RELATED_IMAGE_TFPROXY" + PrepackTensorflowName = "TENSORFLOW_SERVER" + PrepackSklearnName = "SKLEARN_SERVER" + PrepackXgboostName = "XGBOOST_SERVER" + PrepackMlflowName = "MLFLOW_SERVER" + PrepackTritonName = "TRITON_SERVER" ) const PredictorServerConfigMapKeyName = "predictor_servers" @@ -34,15 +30,14 @@ type PredictorImageConfig struct { } type PredictorServerConfig struct { - Tensorflow bool `json:"tensorflow,omitempty"` - TensorflowImage string `json:"tfImage,omitempty"` - RestConfig PredictorImageConfig `json:"rest,omitempty"` - GrpcConfig PredictorImageConfig `json:"grpc,omitempty"` - Protocols PredictorProtocolsConfig `json:"protocols,omitempty"` + Protocols map[Protocol]PredictorImageConfig `json:"protocols"` } -func (p *PredictorServerConfig) PrepackImageName(mlDep *SeldonDeploymentSpec, pu *PredictiveUnit) string { - imageConfig := p.PrepackImageConfig(mlDep, pu) +func (p *PredictorServerConfig) PrepackImageName(protocol Protocol, pu *PredictiveUnit) string { + if string(protocol) == "" { + protocol = ProtocolSeldon + } + imageConfig := p.PrepackImageConfig(protocol) if imageConfig == nil { return "" @@ -55,32 +50,12 @@ func (p *PredictorServerConfig) PrepackImageName(mlDep *SeldonDeploymentSpec, pu return imageConfig.ContainerImage } -func (p *PredictorServerConfig) PrepackImageConfig(mlDep *SeldonDeploymentSpec, pu *PredictiveUnit) *PredictorImageConfig { - // Check if protocol images are defined - switch mlDep.Protocol { - case ProtocolSeldon: - if p.Protocols.Seldon != nil { - return p.Protocols.Seldon - } - case ProtocolTensorflow: - if p.Protocols.Tensorflow != nil { - return p.Protocols.Tensorflow - } - case ProtocolKfserving: - if p.Protocols.KFServing != nil { - return p.Protocols.KFServing - } - } - - // Otherwise fallback to legacy config - switch pu.Endpoint.Type { - case REST: - return &p.RestConfig - case GRPC: - return &p.GrpcConfig +func (p *PredictorServerConfig) PrepackImageConfig(protocol Protocol) *PredictorImageConfig { + if im, ok := p.Protocols[protocol]; ok { + return &im //do something here + } else { + return nil } - - return nil } type PredictorProtocolsConfig struct { @@ -90,59 +65,52 @@ type PredictorProtocolsConfig struct { } var ( - ControllerConfigMapName = "seldon-config" - envSklearnServerRestImageRelated = os.Getenv(EnvSklearnServerImageRestRelated) - envSklearnServerGrpcImageRelated = os.Getenv(EnvSklearnserverImageGrpcRelated) - envXgboostServerRestImageRelated = os.Getenv(EnvXgboostserverImageRestRelated) - envXgboostServerGrpcImageRelated = os.Getenv(EnvXgboostserverImageGrpcRelated) - envMlflowServerRestImageRelated = os.Getenv(EnvMlflowserverImageRestRelated) - envMlflowServerGrpcImageRelated = os.Getenv(EnvMlflowserverImageGrpcRelated) - envTfserverServerImageRelated = os.Getenv(EnvTensorflowImageRelated) - envTfproxyServerRestImageRelated = os.Getenv(EnvTfproxyImageRestRelated) - envTfproxyServerGrpcImageRelated = os.Getenv(EnvTfproxyImageGrpcRelated) - relatedImageConfig = map[string]PredictorServerConfig{} + ControllerConfigMapName = "seldon-config" + envSklearnServerImageRelated = os.Getenv(EnvSklearnServerImageRelated) + envXgboostServerImageRelated = os.Getenv(EnvXgboostserverImageRelated) + envMlflowServerImageRelated = os.Getenv(EnvMlflowserverImageRelated) + envTfserverServerImageRelated = os.Getenv(EnvTensorflowImageRelated) + envTfproxyServerImageRelated = os.Getenv(EnvTfproxyImageRelated) + relatedImageConfig = map[string]PredictorServerConfig{} ) func init() { - if envSklearnServerRestImageRelated != "" { + if envSklearnServerImageRelated != "" { relatedImageConfig[PrepackSklearnName] = PredictorServerConfig{ - RestConfig: PredictorImageConfig{ - ContainerImage: envSklearnServerRestImageRelated, - }, - GrpcConfig: PredictorImageConfig{ - ContainerImage: envSklearnServerGrpcImageRelated, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: { + ContainerImage: envSklearnServerImageRelated, + }, }, } } - if envXgboostServerRestImageRelated != "" { + if envXgboostServerImageRelated != "" { relatedImageConfig[PrepackXgboostName] = PredictorServerConfig{ - RestConfig: PredictorImageConfig{ - ContainerImage: envXgboostServerRestImageRelated, - }, - GrpcConfig: PredictorImageConfig{ - ContainerImage: envXgboostServerGrpcImageRelated, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: { + ContainerImage: envXgboostServerImageRelated, + }, }, } } - if envMlflowServerRestImageRelated != "" { + if envMlflowServerImageRelated != "" { relatedImageConfig[PrepackMlflowName] = PredictorServerConfig{ - RestConfig: PredictorImageConfig{ - ContainerImage: envMlflowServerRestImageRelated, - }, - GrpcConfig: PredictorImageConfig{ - ContainerImage: envMlflowServerGrpcImageRelated, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: { + ContainerImage: envMlflowServerImageRelated, + }, }, } } if envTfserverServerImageRelated != "" { relatedImageConfig[PrepackTensorflowName] = PredictorServerConfig{ - Tensorflow: true, - TensorflowImage: envTfserverServerImageRelated, - RestConfig: PredictorImageConfig{ - ContainerImage: envTfproxyServerRestImageRelated, - }, - GrpcConfig: PredictorImageConfig{ - ContainerImage: envTfproxyServerGrpcImageRelated, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolTensorflow: { + ContainerImage: envTfserverServerImageRelated, + }, + ProtocolSeldon: { + ContainerImage: envTfproxyServerImageRelated, + }, }, } } @@ -200,21 +168,3 @@ func getPrepackServerConfigWithRelated(serverName string, relatedImages map[stri func GetPrepackServerConfig(serverName string) *PredictorServerConfig { return getPrepackServerConfigWithRelated(serverName, relatedImageConfig) } - -// SetImageNameForPrepackContainer: DEPRECATED. Use serverConfig.PrepackImageName() instead -func SetImageNameForPrepackContainer(pu *PredictiveUnit, c *corev1.Container, serverConfig *PredictorServerConfig) { - // Add image: ignore version if empty - if c.Image == "" { - if pu.Endpoint.Type == REST { - c.Image = serverConfig.RestConfig.ContainerImage - if serverConfig.RestConfig.DefaultImageVersion != "" { - c.Image = c.Image + ":" + serverConfig.RestConfig.DefaultImageVersion - } - } else { - c.Image = serverConfig.GrpcConfig.ContainerImage - if serverConfig.GrpcConfig.DefaultImageVersion != "" { - c.Image = c.Image + ":" + serverConfig.GrpcConfig.DefaultImageVersion - } - } - } -} diff --git a/operator/apis/machinelearning.seldon.io/v1/prepack_test.go b/operator/apis/machinelearning.seldon.io/v1/prepack_test.go index 4df5ea3700..5c95421ff1 100644 --- a/operator/apis/machinelearning.seldon.io/v1/prepack_test.go +++ b/operator/apis/machinelearning.seldon.io/v1/prepack_test.go @@ -19,37 +19,45 @@ func TestImageSetNormal(t *testing.T) { "GRPC image with version": { pu: &PredictiveUnit{Endpoint: &Endpoint{Type: GRPC}}, config: &PredictorServerConfig{ - GrpcConfig: PredictorImageConfig{ContainerImage: "a", DefaultImageVersion: "1"}, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: PredictorImageConfig{ContainerImage: "a", DefaultImageVersion: "1"}, + }, }, desiredImageName: "a:1", }, "GRPC image with no version": { pu: &PredictiveUnit{Endpoint: &Endpoint{Type: GRPC}}, config: &PredictorServerConfig{ - GrpcConfig: PredictorImageConfig{ContainerImage: "a"}, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: PredictorImageConfig{ContainerImage: "a", DefaultImageVersion: "1"}, + }, }, - desiredImageName: "a", + desiredImageName: "a:1", }, "REST image with version": { pu: &PredictiveUnit{Endpoint: &Endpoint{Type: REST}}, config: &PredictorServerConfig{ - RestConfig: PredictorImageConfig{ContainerImage: "a", DefaultImageVersion: "1"}, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: PredictorImageConfig{ContainerImage: "a", DefaultImageVersion: "1"}, + }, }, desiredImageName: "a:1", }, "REST image with no version": { pu: &PredictiveUnit{Endpoint: &Endpoint{Type: REST}}, config: &PredictorServerConfig{ - RestConfig: PredictorImageConfig{ContainerImage: "a"}, + Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: PredictorImageConfig{ContainerImage: "a", DefaultImageVersion: "1"}, + }, }, - desiredImageName: "a", + desiredImageName: "a:1", }, } for name, scenario := range scenarios { t.Logf("Scenario: %s", name) con := &corev1.Container{} - SetImageNameForPrepackContainer(scenario.pu, con, scenario.config) + con.Image = scenario.config.PrepackImageName(ProtocolSeldon, scenario.pu) g.Expect(con.Image).To(Equal(scenario.desiredImageName)) } } @@ -67,9 +75,11 @@ func TestGetPredictorConfig(t *testing.T) { }{ "related image sklearn": { - serverName: PrepackSklearnName, - relatedImageMap: map[string]PredictorServerConfig{PrepackSklearnName: {RestConfig: PredictorImageConfig{ContainerImage: "a"}}}, - desiredConfig: PredictorServerConfig{RestConfig: PredictorImageConfig{ContainerImage: "a"}}, + serverName: PrepackSklearnName, + relatedImageMap: map[string]PredictorServerConfig{PrepackSklearnName: {Protocols: map[Protocol]PredictorImageConfig{ + ProtocolSeldon: {ContainerImage: "a"}}}, + }, + desiredConfig: PredictorServerConfig{Protocols: map[Protocol]PredictorImageConfig{ProtocolSeldon: {ContainerImage: "a"}}}, }, "default image sklearn": { serverName: PrepackSklearnName, @@ -84,66 +94,6 @@ func TestGetPredictorConfig(t *testing.T) { } } -func TestPredictorServerConfigPrepackImageConfig(t *testing.T) { - g := NewGomegaWithT(t) - - tests := []struct { - config *PredictorServerConfig - protocol Protocol - endpointType EndpointType - expected *PredictorImageConfig - }{ - { - config: &PredictorServerConfig{ - RestConfig: PredictorImageConfig{ContainerImage: "rest"}, - GrpcConfig: PredictorImageConfig{ContainerImage: "grpc"}, - }, - protocol: ProtocolSeldon, - endpointType: REST, - expected: &PredictorImageConfig{ContainerImage: "rest"}, - }, - { - config: &PredictorServerConfig{ - RestConfig: PredictorImageConfig{ContainerImage: "rest"}, - GrpcConfig: PredictorImageConfig{ContainerImage: "grpc"}, - }, - protocol: ProtocolSeldon, - endpointType: GRPC, - expected: &PredictorImageConfig{ContainerImage: "grpc"}, - }, - { - config: &PredictorServerConfig{ - RestConfig: PredictorImageConfig{ContainerImage: "rest"}, - GrpcConfig: PredictorImageConfig{ContainerImage: "grpc"}, - }, - protocol: ProtocolKfserving, - endpointType: GRPC, - expected: &PredictorImageConfig{ContainerImage: "grpc"}, - }, - { - config: &PredictorServerConfig{ - RestConfig: PredictorImageConfig{ContainerImage: "rest"}, - GrpcConfig: PredictorImageConfig{ContainerImage: "grpc"}, - Protocols: PredictorProtocolsConfig{ - KFServing: &PredictorImageConfig{ContainerImage: "kfserving"}, - }, - }, - protocol: ProtocolKfserving, - endpointType: GRPC, - expected: &PredictorImageConfig{ContainerImage: "kfserving"}, - }, - } - - for _, test := range tests { - mlDep := &SeldonDeploymentSpec{Protocol: test.protocol} - pu := &PredictiveUnit{Endpoint: &Endpoint{Type: test.endpointType}} - - imageConfig := test.config.PrepackImageConfig(mlDep, pu) - - g.Expect(imageConfig).To(Equal(test.expected)) - } -} - func TestPredictorServerConfigPrepackImageName(t *testing.T) { g := NewGomegaWithT(t) @@ -179,12 +129,11 @@ func TestPredictorServerConfigPrepackImageName(t *testing.T) { for _, test := range tests { p := &PredictorServerConfig{} if test.imageConfig != nil { - p.RestConfig = *test.imageConfig + p.Protocols = map[Protocol]PredictorImageConfig{ProtocolSeldon: *test.imageConfig} } - mlDep := &SeldonDeploymentSpec{} pu := &PredictiveUnit{Endpoint: &Endpoint{Type: REST}} - image := p.PrepackImageName(mlDep, pu) + image := p.PrepackImageName(ProtocolSeldon, pu) g.Expect(image).To(Equal(test.expected)) } diff --git a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_types.go b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_types.go index b6dd20de7b..20491f2ca8 100644 --- a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_types.go +++ b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_types.go @@ -53,6 +53,8 @@ const ( PODINFO_VOLUME_PATH = "/etc/podinfo" ENV_PREDICTIVE_UNIT_SERVICE_PORT = "PREDICTIVE_UNIT_SERVICE_PORT" + ENV_PREDICTIVE_UNIT_HTTP_SERVICE_PORT = "PREDICTIVE_UNIT_HTTP_SERVICE_PORT" + ENV_PREDICTIVE_UNIT_GRPC_SERVICE_PORT = "PREDICTIVE_UNIT_GRPC_SERVICE_PORT" ENV_PREDICTIVE_UNIT_SERVICE_PORT_METRICS = "PREDICTIVE_UNIT_METRICS_SERVICE_PORT" ENV_PREDICTIVE_UNIT_METRICS_ENDPOINT = "PREDICTIVE_UNIT_METRICS_ENDPOINT" ENV_PREDICTIVE_UNIT_METRICS_PORT_NAME = "PREDICTIVE_UNIT_METRICS_PORT_NAME" @@ -375,6 +377,8 @@ type Endpoint struct { ServiceHost string `json:"service_host,omitempty" protobuf:"string,1,opt,name=service_host"` ServicePort int32 `json:"service_port,omitempty" protobuf:"int32,2,opt,name=service_port"` Type EndpointType `json:"type,omitempty" protobuf:"int,3,opt,name=type"` + HttpPort int32 `json:"httpPort,omitempty" protobuf:"int32,4,opt,name=httpPort"` + GrpcPort int32 `json:"grpcPort,omitempty" protobuf:"int32,5,opt,name=grpcPort"` } type ParmeterType string diff --git a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go index a0bdfd0df5..73eb3e8812 100644 --- a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go +++ b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook.go @@ -38,7 +38,8 @@ var ( seldondeploymentlog = logf.Log.WithName("seldondeployment") ControllerNamespace = GetEnv("POD_NAMESPACE", "seldon-system") C client.Client - envPredictiveUnitServicePort = os.Getenv(ENV_PREDICTIVE_UNIT_SERVICE_PORT) + envPredictiveUnitHttpServicePort = os.Getenv(ENV_PREDICTIVE_UNIT_HTTP_SERVICE_PORT) + envPredictiveUnitGrpcServicePort = os.Getenv(ENV_PREDICTIVE_UNIT_GRPC_SERVICE_PORT) envPredictiveUnitServicePortMetrics = os.Getenv(ENV_PREDICTIVE_UNIT_SERVICE_PORT_METRICS) envPredictiveUnitMetricsPortName = GetEnv(ENV_PREDICTIVE_UNIT_METRICS_PORT_NAME, constants.DefaultMetricsPortName) ) @@ -122,26 +123,21 @@ func addMetricsPortAndIncrement(nextMetricsPortNum *int32, con *corev1.Container } func (r *SeldonDeploymentSpec) setContainerPredictiveUnitDefaults(compSpecIdx int, - portNum int32, nextMetricsPortNum *int32, mldepName string, namespace string, + portNumHttp int32, portNumGrpc int32, nextMetricsPortNum *int32, mldepName string, namespace string, p *PredictorSpec, pu *PredictiveUnit, con *corev1.Container) { if pu.Endpoint == nil { - if r.Transport == TransportGrpc { - pu.Endpoint = &Endpoint{Type: GRPC} - } else { - pu.Endpoint = &Endpoint{Type: REST} - } + pu.Endpoint = &Endpoint{} } - var portType string - if pu.Endpoint.Type == GRPC { - portType = constants.GrpcPortName - } else { - portType = constants.HttpPortName + + existingHttpPort := GetPort(constants.HttpPortName, con.Ports) + if existingHttpPort != nil { + portNumHttp = existingHttpPort.ContainerPort } - existingPort := GetPort(portType, con.Ports) - if existingPort != nil { - portNum = existingPort.ContainerPort + existingGrpcPort := GetPort(constants.GrpcPortName, con.Ports) + if existingGrpcPort != nil { + portNumGrpc = existingGrpcPort.ContainerPort } volFound := false @@ -183,22 +179,46 @@ func (r *SeldonDeploymentSpec) setContainerPredictiveUnitDefaults(compSpecIdx in containerServiceValue := GetContainerServiceName(mldepName, *p, con) pu.Endpoint.ServiceHost = containerServiceValue + "." + namespace + constants.DNSClusterLocalSuffix } - pu.Endpoint.ServicePort = portNum + + // Backwards compatibility. We set this to grpc port if that is specified otherwise go with http port + // The executor still uses this port to check for readiness and its needed for backwards compatibility + // for old images that only have 1 port for http or grpc open + // TODO: deprecate and remove and fix executor + if pu.Endpoint.Type == GRPC || r.Transport == TransportGrpc { + pu.Endpoint.ServicePort = portNumGrpc + } else { + pu.Endpoint.ServicePort = portNumHttp + } + + pu.Endpoint.HttpPort = portNumHttp + pu.Endpoint.GrpcPort = portNumGrpc } func (r *SeldonDeploymentSpec) DefaultSeldonDeployment(mldepName string, namespace string) { - var firstPuPortNum int32 = constants.FirstPortNumber - if envPredictiveUnitServicePort != "" { - portNum, err := strconv.Atoi(envPredictiveUnitServicePort) + var firstHttpPuPortNum int32 = constants.FirstHttpPortNumber + + if envPredictiveUnitHttpServicePort != "" { + portNum, err := strconv.Atoi(envPredictiveUnitHttpServicePort) + if err != nil { + seldondeploymentlog.Error(err, "Failed to decode predictive unit service port will use default", "envar", ENV_PREDICTIVE_UNIT_HTTP_SERVICE_PORT, "value", envPredictiveUnitHttpServicePort) + } else { + firstHttpPuPortNum = int32(portNum) + } + } + nextHttpPortNum := firstHttpPuPortNum + + var firstGrpcPuPortNum int32 = constants.FirstGrpcPortNumber + if envPredictiveUnitGrpcServicePort != "" { + portNum, err := strconv.Atoi(envPredictiveUnitGrpcServicePort) if err != nil { - seldondeploymentlog.Error(err, "Failed to decode predictive unit service port will use default", "envar", ENV_PREDICTIVE_UNIT_SERVICE_PORT, "value", envPredictiveUnitServicePort) + seldondeploymentlog.Error(err, "Failed to decode grpc predictive unit service port will use default", "envar", ENV_PREDICTIVE_UNIT_GRPC_SERVICE_PORT, "value", envPredictiveUnitGrpcServicePort) } else { - firstPuPortNum = int32(portNum) + firstGrpcPuPortNum = int32(portNum) } } - nextPortNum := firstPuPortNum + nextGrpcPortNum := firstGrpcPuPortNum var firstMetricsPuPortNum int32 = constants.FirstMetricsPortNumber if envPredictiveUnitServicePortMetrics != "" { @@ -210,7 +230,8 @@ func (r *SeldonDeploymentSpec) DefaultSeldonDeployment(mldepName string, namespa } } nextMetricsPortNum := firstMetricsPuPortNum - portMap := map[string]int32{} + portMapHttp := map[string]int32{} + portMapGrpc := map[string]int32{} for i := 0; i < len(r.Predictors); i++ { p := r.Predictors[i] @@ -232,13 +253,16 @@ func (r *SeldonDeploymentSpec) DefaultSeldonDeployment(mldepName string, namespa for k := 0; k < len(cSpec.Spec.Containers); k++ { con := &cSpec.Spec.Containers[k] - getUpdatePortNumMap(con.Name, &nextPortNum, portMap) - portNum := portMap[con.Name] + getUpdatePortNumMap(con.Name, &nextHttpPortNum, portMapHttp) + httpPortNum := portMapHttp[con.Name] + + getUpdatePortNumMap(con.Name, &nextGrpcPortNum, portMapGrpc) + grpcPortNum := portMapGrpc[con.Name] pu := GetPredictiveUnit(&p.Graph, con.Name) if pu != nil { - r.setContainerPredictiveUnitDefaults(j, portNum, &nextMetricsPortNum, mldepName, namespace, &p, pu, con) + r.setContainerPredictiveUnitDefaults(j, httpPortNum, grpcPortNum, &nextMetricsPortNum, mldepName, namespace, &p, pu, con) } } } @@ -266,16 +290,19 @@ func (r *SeldonDeploymentSpec) DefaultSeldonDeployment(mldepName string, namespa } } - getUpdatePortNumMap(pu.Name, &nextPortNum, portMap) - portNum := portMap[pu.Name] + getUpdatePortNumMap(pu.Name, &nextHttpPortNum, portMapHttp) + httpPortNum := portMapHttp[pu.Name] + + getUpdatePortNumMap(con.Name, &nextGrpcPortNum, portMapGrpc) + grpcPortNum := portMapGrpc[con.Name] - r.setContainerPredictiveUnitDefaults(0, portNum, &nextMetricsPortNum, mldepName, namespace, &p, pu, con) + r.setContainerPredictiveUnitDefaults(0, httpPortNum, grpcPortNum, &nextMetricsPortNum, mldepName, namespace, &p, pu, con) //Only set image default for non tensorflow graphs if r.Protocol != ProtocolTensorflow { serverConfig := GetPrepackServerConfig(string(*pu.Implementation)) if serverConfig != nil { if con.Image == "" { - con.Image = serverConfig.PrepackImageName(r, pu) + con.Image = serverConfig.PrepackImageName(r.Protocol, pu) } } } diff --git a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go index 5b18ec7871..986d107f07 100644 --- a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go +++ b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_webhook_test.go @@ -75,58 +75,70 @@ func setupTestConfigMap() error { var configs = map[string]string{ "predictor_servers": `{ - "TENSORFLOW_SERVER": { - "tensorflow": true, - "tfImage": "tensorflow/serving:latest", - "rest": { - "image": "seldonio/tfserving-proxy_rest", - "defaultImageVersion": "0.7" - }, - "grpc": { - "image": "seldonio/tfserving-proxy_grpc", - "defaultImageVersion": "0.7" - } - }, - "SKLEARN_SERVER": { - "rest": { - "image": "seldonio/sklearnserver_rest", - "defaultImageVersion": "0.2" - }, - "grpc": { - "image": "seldonio/sklearnserver_grpc", - "defaultImageVersion": "0.2" - } - }, - "XGBOOST_SERVER": { - "rest": { - "image": "seldonio/xgboostserver_rest", - "defaultImageVersion": "0.2" - }, - "grpc": { - "image": "seldonio/xgboostserver_grpc", - "defaultImageVersion": "0.2" - } - }, - "MLFLOW_SERVER": { - "rest": { - "image": "seldonio/mlflowserver_rest", - "defaultImageVersion": "0.2" - }, - "grpc": { - "image": "seldonio/mlflowserver_grpc", - "defaultImageVersion": "0.2" - } - }, - "CUSTOM_SERVER": { - "rest": { - "image": "custom_rest", - "defaultImageVersion": "0.2" - }, - "grpc": { - "image": "custom_grpc", - "defaultImageVersion": "0.2" - } - } + "TENSORFLOW_SERVER": { + "protocols" : { + "tensorflow": { + "image": "tensorflow/serving", + "defaultImageVersion": "2.1.0" + }, + "seldon": { + "image": "seldonio/tfserving-proxy", + "defaultImageVersion": "1.3.0-dev" + } + } + }, + "SKLEARN_SERVER": { + "protocols" : { + "seldon": { + "image": "seldonio/sklearnserver", + "defaultImageVersion": "1.3.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } + } + }, + "XGBOOST_SERVER": { + "protocols" : { + "seldon": { + "image": "seldonio/xgboostserver", + "defaultImageVersion": "1.3.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } + } + }, + "MLFLOW_SERVER": { + "protocols" : { + "seldon": { + "image": "seldonio/mlflowserver", + "defaultImageVersion": "1.3.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } + } + }, + "TRITON_SERVER": { + "protocols" : { + "kfserving": { + "image": "nvcr.io/nvidia/tritonserver", + "defaultImageVersion": "20.08-py3" + } + } + }, + "CUSTOM_SERVER": { + "protocols" : { + "seldon": { + "image": "custom", + "defaultImageVersion": "0.1" + } + } + } }`, } @@ -199,7 +211,7 @@ func TestValidateBadTransport(t *testing.T) { g.Expect(err).ToNot(BeNil()) serr := err.(*errors.StatusError) g.Expect(serr.Status().Code).To(Equal(int32(422))) - g.Expect(len(serr.Status().Details.Causes)).To(Equal(2)) + g.Expect(len(serr.Status().Details.Causes)).To(Equal(1)) g.Expect(serr.Status().Details.Causes[0].Type).To(Equal(v12.CauseTypeFieldValueInvalid)) g.Expect(serr.Status().Details.Causes[0].Field).To(Equal("spec")) } @@ -370,9 +382,8 @@ func TestDefaultSingleContainer(t *testing.T) { // Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) // Volumes volFound := false @@ -431,15 +442,13 @@ func TestMetricsPortAddedTwoContainers(t *testing.T) { //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) pu = GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier2") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber + 1)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber + 1)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) // Volumes volFound := false @@ -514,17 +523,15 @@ func TestMetricsPortAddedTwoComponentSpecsTwoContainers(t *testing.T) { //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) pu = GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier2") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber + 1)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber + 1)) containerServiceValue := GetContainerServiceName(name, spec.Predictors[0], &spec.Predictors[0].ComponentSpecs[1].Spec.Containers[0]) dnsName := containerServiceValue + "." + namespace + constants.DNSClusterLocalSuffix g.Expect(pu.Endpoint.ServiceHost).To(Equal(dnsName)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) // Volumes volFound := false @@ -574,7 +581,6 @@ func TestOverrideMetricsPortName(t *testing.T) { // Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) g.Expect(*pu.Type).To(Equal(MODEL)) } @@ -617,7 +623,6 @@ func TestPortUseExisting(t *testing.T) { g.Expect(pu).ToNot(BeNil()) g.Expect(pu.Endpoint.ServicePort).To(Equal(containerPortAPI)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) } func TestMetricsPortAddedToPrepacked(t *testing.T) { @@ -648,7 +653,7 @@ func TestMetricsPortAddedToPrepacked(t *testing.T) { //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) g.Expect(pu.Endpoint.Type).To(Equal(REST)) } @@ -679,9 +684,8 @@ func TestPredictorProtocolGrpc(t *testing.T) { //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstGrpcPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(GRPC)) } func TestPrepackedWithExistingContainer(t *testing.T) { @@ -719,14 +723,13 @@ func TestPrepackedWithExistingContainer(t *testing.T) { g.Expect(metricPort.ContainerPort).To(Equal(constants.FirstMetricsPortNumber)) // image set from configMap - g.Expect(spec.Predictors[0].ComponentSpecs[0].Spec.Containers[0].Image).To(Equal("seldonio/tfserving-proxy_grpc:0.7")) + g.Expect(spec.Predictors[0].ComponentSpecs[0].Spec.Containers[0].Image).To(Equal("seldonio/tfserving-proxy:1.3.0-dev")) //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstGrpcPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(GRPC)) } func TestPrepackedWithCustom(t *testing.T) { @@ -754,14 +757,13 @@ func TestPrepackedWithCustom(t *testing.T) { g.Expect(metricPort.Name).To(Equal(envPredictiveUnitMetricsPortName)) // image set from configMap - g.Expect(spec.Predictors[0].ComponentSpecs[0].Spec.Containers[0].Image).To(Equal("custom_grpc:0.2")) + g.Expect(spec.Predictors[0].ComponentSpecs[0].Spec.Containers[0].Image).To(Equal("custom:0.1")) //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstGrpcPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(GRPC)) } func TestPrepackedWithExistingContainerAndImage(t *testing.T) { @@ -807,9 +809,8 @@ func TestPrepackedWithExistingContainerAndImage(t *testing.T) { //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstGrpcPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(GRPC)) } func TestMetricsPortAddedToTwoPrepacked(t *testing.T) { @@ -847,15 +848,13 @@ func TestMetricsPortAddedToTwoPrepacked(t *testing.T) { //Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) pu = GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier2") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstPortNumber + 1)) + g.Expect(pu.Endpoint.ServicePort).To(Equal(constants.FirstHttpPortNumber + 1)) g.Expect(pu.Endpoint.ServiceHost).To(Equal(constants.DNSLocalHost)) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) } func TestDefaultPrepackagedServerType(t *testing.T) { @@ -880,7 +879,6 @@ func TestDefaultPrepackagedServerType(t *testing.T) { // Graph pu := GetPredictiveUnit(&spec.Predictors[0].Graph, "classifier") g.Expect(pu).ToNot(BeNil()) - g.Expect(pu.Endpoint.Type).To(Equal(REST)) g.Expect(*pu.Type).To(Equal(MODEL)) } diff --git a/operator/apis/machinelearning.seldon.io/v1/zz_generated.deepcopy.go b/operator/apis/machinelearning.seldon.io/v1/zz_generated.deepcopy.go index 4efa65a4db..75e46d81a5 100644 --- a/operator/apis/machinelearning.seldon.io/v1/zz_generated.deepcopy.go +++ b/operator/apis/machinelearning.seldon.io/v1/zz_generated.deepcopy.go @@ -225,9 +225,13 @@ func (in *PredictorProtocolsConfig) DeepCopy() *PredictorProtocolsConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PredictorServerConfig) DeepCopyInto(out *PredictorServerConfig) { *out = *in - out.RestConfig = in.RestConfig - out.GrpcConfig = in.GrpcConfig - in.Protocols.DeepCopyInto(&out.Protocols) + if in.Protocols != nil { + in, out := &in.Protocols, &out.Protocols + *out = make(map[Protocol]PredictorImageConfig, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PredictorServerConfig. diff --git a/operator/config/crd/bases/machinelearning.seldon.io_seldondeployments.yaml b/operator/config/crd/bases/machinelearning.seldon.io_seldondeployments.yaml index c28f175848..b6de8f37d0 100644 --- a/operator/config/crd/bases/machinelearning.seldon.io_seldondeployments.yaml +++ b/operator/config/crd/bases/machinelearning.seldon.io_seldondeployments.yaml @@ -27,10 +27,14 @@ spec: description: SeldonDeployment is the Schema for the seldondeployments API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -65,30 +69,56 @@ spec: type: integer metrics: items: - description: MetricSpec specifies how to scale based on a single metric (only `type` and one other matching field should be set at once). + description: MetricSpec specifies how to scale based + on a single metric (only `type` and one other matching + field should be set at once). properties: external: - description: external refers to a global metric that is not associated with any Kubernetes object. It allows autoscaling based on information coming from components running outside of cluster (for example length of queue in cloud messaging service, or QPS from loadbalancer running outside of cluster). + description: external refers to a global metric + that is not associated with any Kubernetes object. + It allows autoscaling based on information coming + from components running outside of cluster (for + example length of queue in cloud messaging service, + or QPS from loadbalancer running outside of + cluster). properties: metricName: - description: metricName is the name of the metric in question. + description: metricName is the name of the + metric in question. type: string metricSelector: - description: metricSelector is used to identify a specific time series within a given metric. + description: metricSelector is used to identify + a specific time series within a given metric. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -100,55 +130,91 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object targetAverageValue: anyOf: - type: integer - type: string - description: targetAverageValue is the target per-pod value of global metric (as a quantity). Mutually exclusive with TargetValue. + description: targetAverageValue is the target + per-pod value of global metric (as a quantity). + Mutually exclusive with TargetValue. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true targetValue: anyOf: - type: integer - type: string - description: targetValue is the target value of the metric (as a quantity). Mutually exclusive with TargetAverageValue. + description: targetValue is the target value + of the metric (as a quantity). Mutually + exclusive with TargetAverageValue. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: - metricName type: object object: - description: object refers to a metric describing a single kubernetes object (for example, hits-per-second on an Ingress object). + description: object refers to a metric describing + a single kubernetes object (for example, hits-per-second + on an Ingress object). properties: averageValue: anyOf: - type: integer - type: string - description: averageValue is the target value of the average of the metric across all relevant pods (as a quantity) + description: averageValue is the target value + of the average of the metric across all + relevant pods (as a quantity) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true metricName: - description: metricName is the name of the metric in question. + description: metricName is the name of the + metric in question. type: string selector: - description: selector is the string-encoded form of a standard kubernetes label selector for the given metric When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping When unset, just the metricName will be used to gather metrics. + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping + When unset, just the metricName will be + used to gather metrics. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -160,20 +226,28 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object target: - description: target is the described Kubernetes object. + description: target is the described Kubernetes + object. properties: apiVersion: description: API version of the referent type: string kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + description: 'Kind of the referent; More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + description: 'Name of the referent; More + info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string required: - kind @@ -183,7 +257,8 @@ spec: anyOf: - type: integer - type: string - description: targetValue is the target value of the metric (as a quantity). + description: targetValue is the target value + of the metric (as a quantity). pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: @@ -192,27 +267,54 @@ spec: - targetValue type: object pods: - description: pods refers to a metric describing each pod in the current scale target (for example, transactions-processed-per-second). The values will be averaged together before being compared to the target value. + description: pods refers to a metric describing + each pod in the current scale target (for example, + transactions-processed-per-second). The values + will be averaged together before being compared + to the target value. properties: metricName: - description: metricName is the name of the metric in question + description: metricName is the name of the + metric in question type: string selector: - description: selector is the string-encoded form of a standard kubernetes label selector for the given metric When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping When unset, just the metricName will be used to gather metrics. + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping + When unset, just the metricName will be + used to gather metrics. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -224,14 +326,21 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object targetAverageValue: anyOf: - type: integer - type: string - description: targetAverageValue is the target value of the average of the metric across all relevant pods (as a quantity) + description: targetAverageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: @@ -239,27 +348,44 @@ spec: - targetAverageValue type: object resource: - description: resource refers to a resource metric (such as those specified in requests and limits) known to Kubernetes describing each pod in the current scale target (e.g. CPU or memory). Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the "pods" source. + description: resource refers to a resource metric + (such as those specified in requests and limits) + known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have + special scaling options on top of those available + to normal per-pod metrics using the "pods" source. properties: name: - description: name is the name of the resource in question. + description: name is the name of the resource + in question. type: string targetAverageUtilization: - description: targetAverageUtilization is the target value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. + description: targetAverageUtilization is the + target value of the average of the resource + metric across all relevant pods, represented + as a percentage of the requested value of + the resource for the pods. format: int32 type: integer targetAverageValue: anyOf: - type: integer - type: string - description: targetAverageValue is the target value of the average of the resource metric across all relevant pods, as a raw value (instead of as a percentage of the request), similar to the "pods" metric source type. + description: targetAverageValue is the target + value of the average of the resource metric + across all relevant pods, as a raw value + (instead of as a percentage of the request), + similar to the "pods" metric source type. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: - name type: object type: - description: type is the type of metric source. It should be one of "Object", "Pods" or "Resource", each mapping to a matching field in the object. + description: type is the type of metric source. It + should be one of "Object", "Pods" or "Resource", + each mapping to a matching field in the object. type: string required: - type @@ -272,34 +398,60 @@ spec: - maxReplicas type: object kedaSpec: - description: SeldonScaledObjectSpec is the spec for a KEDA ScaledObject resource + description: SeldonScaledObjectSpec is the spec for a KEDA + ScaledObject resource properties: advanced: - description: AdvancedConfig specifies advance scaling options + description: AdvancedConfig specifies advance scaling + options properties: horizontalPodAutoscalerConfig: - description: HorizontalPodAutoscalerConfig specifies horizontal scale config + description: HorizontalPodAutoscalerConfig specifies + horizontal scale config properties: behavior: - description: HorizontalPodAutoscalerBehavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively). + description: HorizontalPodAutoscalerBehavior + configures the scaling behavior of the target + in both Up and Down directions (scaleUp and + scaleDown fields respectively). properties: scaleDown: - description: scaleDown is scaling policy for scaling Down. If not set, the default value is to allow to scale down to minReplicas pods, with a 300 second stabilization window (i.e., the highest recommendation for the last 300sec is used). + description: scaleDown is scaling policy + for scaling Down. If not set, the default + value is to allow to scale down to minReplicas + pods, with a 300 second stabilization + window (i.e., the highest recommendation + for the last 300sec is used). properties: policies: - description: policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + description: policies is a list of potential + scaling polices which can be used + during scaling. At least one policy + must be specified, otherwise the HPAScalingRules + will be discarded as invalid items: - description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + description: HPAScalingPolicy is a + single policy which must hold true + for a specified past interval. properties: periodSeconds: - description: PeriodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + description: PeriodSeconds specifies + the window of time for which + the policy should hold true. + PeriodSeconds must be greater + than zero and less than or equal + to 1800 (30 min). format: int32 type: integer type: - description: Type is used to specify the scaling policy. + description: Type is used to specify + the scaling policy. type: string value: - description: Value contains the amount of change which is permitted by the policy. It must be greater than zero + description: Value contains the + amount of change which is permitted + by the policy. It must be greater + than zero format: int32 type: integer required: @@ -309,30 +461,64 @@ spec: type: object type: array selectPolicy: - description: selectPolicy is used to specify which policy should be used. If not set, the default value MaxPolicySelect is used. + description: selectPolicy is used to + specify which policy should be used. + If not set, the default value MaxPolicySelect + is used. type: string stabilizationWindowSeconds: - description: 'StabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).' + description: 'StabilizationWindowSeconds + is the number of seconds for which + past recommendations should be considered + while scaling up or scaling down. + StabilizationWindowSeconds must be + greater than or equal to zero and + less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. + the stabilization window is 300 seconds + long).' format: int32 type: integer type: object scaleUp: - description: 'scaleUp is scaling policy for scaling Up. If not set, the default value is the higher of: * increase no more than 4 pods per 60 seconds * double the number of pods per 60 seconds No stabilization is used.' + description: 'scaleUp is scaling policy + for scaling Up. If not set, the default + value is the higher of: * increase no + more than 4 pods per 60 seconds * double + the number of pods per 60 seconds No stabilization + is used.' properties: policies: - description: policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + description: policies is a list of potential + scaling polices which can be used + during scaling. At least one policy + must be specified, otherwise the HPAScalingRules + will be discarded as invalid items: - description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + description: HPAScalingPolicy is a + single policy which must hold true + for a specified past interval. properties: periodSeconds: - description: PeriodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + description: PeriodSeconds specifies + the window of time for which + the policy should hold true. + PeriodSeconds must be greater + than zero and less than or equal + to 1800 (30 min). format: int32 type: integer type: - description: Type is used to specify the scaling policy. + description: Type is used to specify + the scaling policy. type: string value: - description: Value contains the amount of change which is permitted by the policy. It must be greater than zero + description: Value contains the + amount of change which is permitted + by the policy. It must be greater + than zero format: int32 type: integer required: @@ -342,43 +528,82 @@ spec: type: object type: array selectPolicy: - description: selectPolicy is used to specify which policy should be used. If not set, the default value MaxPolicySelect is used. + description: selectPolicy is used to + specify which policy should be used. + If not set, the default value MaxPolicySelect + is used. type: string stabilizationWindowSeconds: - description: 'StabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).' + description: 'StabilizationWindowSeconds + is the number of seconds for which + past recommendations should be considered + while scaling up or scaling down. + StabilizationWindowSeconds must be + greater than or equal to zero and + less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. + the stabilization window is 300 seconds + long).' format: int32 type: integer type: object type: object resourceMetrics: items: - description: ResourceMetricSource indicates how to scale on a resource metric known to Kubernetes, as specified in requests and limits, describing each pod in the current scale target (e.g. CPU or memory). The values will be averaged together before being compared to the target. Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the "pods" source. Only one "target" type should be set. + description: ResourceMetricSource indicates + how to scale on a resource metric known + to Kubernetes, as specified in requests + and limits, describing each pod in the current + scale target (e.g. CPU or memory). The + values will be averaged together before + being compared to the target. Such metrics + are built in to Kubernetes, and have special + scaling options on top of those available + to normal per-pod metrics using the "pods" + source. Only one "target" type should be + set. properties: name: - description: name is the name of the resource in question. + description: name is the name of the resource + in question. type: string target: - description: target specifies the target value for the given metric + description: target specifies the target + value for the given metric properties: averageUtilization: - description: averageUtilization is the target value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. Currently only valid for Resource metric source type + description: averageUtilization is + the target value of the average + of the resource metric across all + relevant pods, represented as a + percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type format: int32 type: integer averageValue: anyOf: - type: integer - type: string - description: averageValue is the target value of the average of the metric across all relevant pods (as a quantity) + description: averageValue is the target + value of the average of the metric + across all relevant pods (as a quantity) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: - description: type represents whether the metric type is Utilization, Value, or AverageValue + description: type represents whether + the metric type is Utilization, + Value, or AverageValue type: string value: anyOf: - type: integer - type: string - description: value is the target value of the metric (as a quantity). + description: value is the target value + of the metric (as a quantity). pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: @@ -407,10 +632,13 @@ spec: type: integer triggers: items: - description: ScaleTriggers reference the scaler that will be used + description: ScaleTriggers reference the scaler that + will be used properties: authenticationRef: - description: ScaledObjectAuthRef points to the TriggerAuthentication object that is used to authenticate the scaler with the environment + description: ScaledObjectAuthRef points to the + TriggerAuthentication object that is used to + authenticate the scaler with the environment properties: name: type: string @@ -441,13 +669,23 @@ spec: anyOf: - type: integer - type: string - description: An eviction is allowed if at most "maxUnavailable" pods in the deployment corresponding to a componentSpec are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. MaxUnavailable and MinAvailable are mutually exclusive. + description: An eviction is allowed if at most "maxUnavailable" + pods in the deployment corresponding to a componentSpec + are unavailable after the eviction, i.e. even in absence + of the evicted pod. For example, one can prevent all + voluntary evictions by specifying 0. MaxUnavailable + and MinAvailable are mutually exclusive. x-kubernetes-int-or-string: true minAvailable: anyOf: - type: integer - type: string - description: An eviction is allowed if at least "minAvailable" pods in the deployment corresponding to a componentSpec will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying "100%". + description: An eviction is allowed if at least "minAvailable" + pods in the deployment corresponding to a componentSpec + will still be available after the eviction, i.e. even + in the absence of the evicted pod. So for example + you can prevent all voluntary evictions by specifying + "100%". x-kubernetes-int-or-string: true type: object replicas: @@ -457,36 +695,79 @@ spec: description: PodSpec is a description of a pod. properties: activeDeadlineSeconds: - description: Optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers. Value must be a positive integer. + description: Optional duration in seconds the pod may + be active on the node relative to StartTime before + the system will actively try to mark it failed and + kill associated containers. Value must be a positive + integer. format: int64 type: integer affinity: description: If specified, the pod's scheduling constraints properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling + rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most + preferred. items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + description: An empty preferred scheduling + term matches all objects with implicit weight + 0 (i.e. it's a no-op). A null preferred + scheduling term matches no objects (i.e. + is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated + with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector + requirements by node's labels. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -496,18 +777,38 @@ spec: type: object type: array matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector + requirements by node's fields. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -518,7 +819,9 @@ spec: type: array type: object weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. format: int32 type: integer required: @@ -527,26 +830,57 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to an update), the + system may or may not try to eventually evict + the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector + terms. The terms are ORed. items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + description: A null or empty node selector + term matches no objects. The requirements + of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector + requirements by node's labels. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -556,18 +890,38 @@ spec: type: object type: array matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector + requirements by node's fields. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -583,32 +937,74 @@ spec: type: object type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest + sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity + term, associated with the corresponding + weight. properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a + set of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the + label key that the selector + applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. items: type: string type: array @@ -620,22 +1016,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. format: int32 type: integer required: @@ -644,26 +1063,64 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to a pod label update), + the system may or may not try to eventually + evict the pod from its node. When there are + multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. + all terms must be satisfied. items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a set + of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label + key that the selector applies + to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. items: type: string type: array @@ -675,16 +1132,34 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey @@ -692,32 +1167,75 @@ spec: type: array type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule + pods to nodes that satisfy the anti-affinity + expressions specified by this field, but it + may choose a node that violates one or more + of the expressions. The node that is most + preferred is the one with the greatest sum + of weights, i.e. for each node that meets + all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity + expressions, etc.), compute a sum by iterating + through the elements of this field and adding + "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most + preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity + term, associated with the corresponding + weight. properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a + set of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the + label key that the selector + applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. items: type: string type: array @@ -729,22 +1247,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. format: int32 type: integer required: @@ -753,26 +1294,64 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: If the anti-affinity requirements + specified by this field are not met at scheduling + time, the pod will not be scheduled onto the + node. If the anti-affinity requirements specified + by this field cease to be met at some point + during pod execution (e.g. due to a pod label + update), the system may or may not try to + eventually evict the pod from its node. When + there are multiple elements, the lists of + nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a set + of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label + key that the selector applies + to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. items: type: string type: array @@ -784,16 +1363,34 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey @@ -802,36 +1399,76 @@ spec: type: object type: object automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. type: boolean containers: - description: List of containers belonging to the pod. Containers cannot currently be added or removed. There must be at least one container in a Pod. Cannot be updated. + description: List of containers belonging to the pod. + Containers cannot currently be added or removed. There + must be at least one container in a Pod. Cannot be + updated. items: - description: A single application container that you want to run within a pod. + description: A single application container that you + want to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The + docker image''s CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable + cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within + a shell. The docker image''s ENTRYPOINT is used + if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a + double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to + set in the container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment + variable present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) + are expanded using the previous defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. The $(VAR_NAME) syntax can + be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment + variable's value. Cannot be used if value + is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -840,56 +1477,84 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the + ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to + select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required + for volumes, optional for env + vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output + format of the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: resource + to select' type: string required: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret + in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret + to select from. Must be a valid + secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the + Secret or its key must be defined type: boolean required: - key @@ -900,72 +1565,127 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source + of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap + must be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret + must be defined type: boolean type: object type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level + config management to default or override container + images in workload controllers like Deployments + and StatefulSets.' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. + description: Actions that the management system + should take in response to container lifecycle + events. Cannot be updated. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -973,64 +1693,114 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The reason for termination is passed to + the handler. The Pod''s termination grace + period countdown begins before the PreStop + hooked is executed. Regardless of the outcome + of the handler, the container will eventually + terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until + the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -1038,31 +1808,44 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -1070,31 +1853,53 @@ spec: type: object type: object livenessProbe: - description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1108,76 +1913,121 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + description: Name of the container specified as + a DNS_LABEL. Each container in a pod must have + a unique name (DNS_LABEL). Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. + description: List of ports to expose from the + container. Exposing a port here gives the system + additional information about the network connections + a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent + that port from being exposed. Any port which + is listening on the default "0.0.0.0" address + inside a container will be accessible from the + network. Cannot be updated. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network + port in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external + port to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort @@ -1188,31 +2038,54 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container service + readiness. Container will be removed from service + endpoints if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1226,54 +2099,77 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Compute Resources required by this + container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -1282,7 +2178,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -1291,107 +2189,213 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'Security options the pod should + run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a + read-only root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label + that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label + that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label + that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label + that applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. type: string type: object type: object startupProbe: - description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'StartupProbe indicates that the + Pod has successfully initialized. If specified, + no other probes are executed until this completes + successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. + This can be used to provide different probe + parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data + or warm a cache, than during steady-state operation. + This cannot be updated. This is a beta feature + enabled by the StartupProbe feature flag. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1405,77 +2409,139 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false type: boolean terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block + devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping + of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside + of the container that the device will + be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of + a persistentVolumeClaim in the pod type: string required: - devicePath @@ -1483,27 +2549,46 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting + of a Volume within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. type: string name: - description: This must match the Name of a Volume. + description: This must match the Name of + a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. type: string required: - mountPath @@ -1511,24 +2596,37 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. type: string required: - name type: object type: array dnsConfig: - description: Specifies the DNS parameters of a pod. Parameters specified here will be merged to the generated DNS configuration based on DNSPolicy. + description: Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated + DNS configuration based on DNSPolicy. properties: nameservers: - description: A list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. Duplicated nameservers will be removed. + description: A list of DNS name server IP addresses. + This will be appended to the base nameservers + generated from DNSPolicy. Duplicated nameservers + will be removed. items: type: string type: array options: - description: A list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Duplicated entries will be removed. Resolution options given in Options will override those that appear in the base DNSPolicy. + description: A list of DNS resolver options. This + will be merged with the base options generated + from DNSPolicy. Duplicated entries will be removed. + Resolution options given in Options will override + those that appear in the base DNSPolicy. items: - description: PodDNSConfigOption defines DNS resolver options of a pod. + description: PodDNSConfigOption defines DNS resolver + options of a pod. properties: name: description: Required. @@ -1538,45 +2636,112 @@ spec: type: object type: array searches: - description: A list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. Duplicated search paths will be removed. + description: A list of DNS search domains for host-name + lookup. This will be appended to the base search + paths generated from DNSPolicy. Duplicated search + paths will be removed. items: type: string type: array type: object dnsPolicy: - description: Set DNS policy for the pod. Defaults to "ClusterFirst". Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'. + description: Set DNS policy for the pod. Defaults to + "ClusterFirst". Valid values are 'ClusterFirstWithHostNet', + 'ClusterFirst', 'Default' or 'None'. DNS parameters + given in DNSConfig will be merged with the policy + selected with DNSPolicy. To have DNS options set along + with hostNetwork, you have to specify DNS policy explicitly + to 'ClusterFirstWithHostNet'. type: string enableServiceLinks: - description: 'EnableServiceLinks indicates whether information about services should be injected into pod''s environment variables, matching the syntax of Docker links. Optional: Defaults to true.' + description: 'EnableServiceLinks indicates whether information + about services should be injected into pod''s environment + variables, matching the syntax of Docker links. Optional: + Defaults to true.' type: boolean ephemeralContainers: - description: List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing pod to perform user-initiated actions such as debugging. This list cannot be specified when creating a pod, and it cannot be modified by updating the pod spec. In order to add an ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature. + description: List of ephemeral containers run in this + pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. + This list cannot be specified when creating a pod, + and it cannot be modified by updating the pod spec. + In order to add an ephemeral container to an existing + pod, use the pod's ephemeralcontainers subresource. + This field is alpha-level and is only honored by servers + that enable the EphemeralContainers feature. items: - description: An EphemeralContainer is a container that may be added temporarily to an existing pod for user-initiated activities such as debugging. Ephemeral containers have no resource or scheduling guarantees, and they will not be restarted when they exit or when a pod is removed or restarted. If an ephemeral container causes a pod to exceed its resource allocation, the pod may be evicted. Ephemeral containers may not be added by directly updating the pod spec. They must be added via the pod's ephemeralcontainers subresource, and they will appear in the pod spec once added. This is an alpha feature enabled by the EphemeralContainers feature flag. + description: An EphemeralContainer is a container + that may be added temporarily to an existing pod + for user-initiated activities such as debugging. + Ephemeral containers have no resource or scheduling + guarantees, and they will not be restarted when + they exit or when a pod is removed or restarted. + If an ephemeral container causes a pod to exceed + its resource allocation, the pod may be evicted. + Ephemeral containers may not be added by directly + updating the pod spec. They must be added via the + pod's ephemeralcontainers subresource, and they + will appear in the pod spec once added. This is + an alpha feature enabled by the EphemeralContainers + feature flag. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The + docker image''s CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable + cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within + a shell. The docker image''s ENTRYPOINT is used + if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a + double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to + set in the container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment + variable present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) + are expanded using the previous defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. The $(VAR_NAME) syntax can + be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment + variable's value. Cannot be used if value + is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -1585,56 +2750,84 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the + ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to + select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required + for volumes, optional for env + vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output + format of the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: resource + to select' type: string required: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret + in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret + to select from. Must be a valid + secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the + Secret or its key must be defined type: boolean required: - key @@ -1645,31 +2838,50 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source + of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap + must be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret + must be defined type: boolean type: object type: object @@ -1678,39 +2890,70 @@ spec: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Lifecycle is not allowed for ephemeral containers. + description: Lifecycle is not allowed for ephemeral + containers. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -1718,64 +2961,114 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The reason for termination is passed to + the handler. The Pod''s termination grace + period countdown begins before the PreStop + hooked is executed. Regardless of the outcome + of the handler, the container will eventually + terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until + the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -1783,31 +3076,44 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -1815,31 +3121,52 @@ spec: type: object type: object livenessProbe: - description: Probes are not allowed for ephemeral containers. + description: Probes are not allowed for ephemeral + containers. properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1853,107 +3180,167 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the ephemeral container specified as a DNS_LABEL. This name must be unique among all containers, init containers and ephemeral containers. + description: Name of the ephemeral container specified + as a DNS_LABEL. This name must be unique among + all containers, init containers and ephemeral + containers. type: string ports: - description: Ports are not allowed for ephemeral containers. + description: Ports are not allowed for ephemeral + containers. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network + port in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external + port to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array readinessProbe: - description: Probes are not allowed for ephemeral containers. + description: Probes are not allowed for ephemeral + containers. properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1967,54 +3354,78 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources already allocated to the pod. + description: Resources are not allowed for ephemeral + containers. Ephemeral containers use spare resources + already allocated to the pod. properties: limits: additionalProperties: @@ -2023,7 +3434,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -2032,107 +3445,202 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: SecurityContext is not allowed for ephemeral containers. + description: SecurityContext is not allowed for + ephemeral containers. properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a + read-only root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label + that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label + that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label + that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label + that applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. type: string type: object type: object startupProbe: - description: Probes are not allowed for ephemeral containers. + description: Probes are not allowed for ephemeral + containers. properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2146,80 +3654,148 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false type: boolean targetContainerName: - description: If set, the name of the container from PodSpec that this ephemeral container targets. The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. If not set then the ephemeral container is run in whatever namespaces are shared for the pod. Note that the container runtime must support this feature. + description: If set, the name of the container + from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces + (IPC, PID, etc) of this container. If not set + then the ephemeral container is run in whatever + namespaces are shared for the pod. Note that + the container runtime must support this feature. type: string terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block + devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping + of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside + of the container that the device will + be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of + a persistentVolumeClaim in the pod type: string required: - devicePath @@ -2227,27 +3803,46 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting + of a Volume within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. type: string name: - description: This must match the Name of a Volume. + description: This must match the Name of + a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. type: string required: - mountPath @@ -2255,16 +3850,24 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. type: string required: - name type: object type: array hostAliases: - description: HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts file if specified. This is only valid for non-hostNetwork pods. + description: HostAliases is an optional list of hosts + and IPs that will be injected into the pod's hosts + file if specified. This is only valid for non-hostNetwork + pods. items: - description: HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file. + description: HostAlias holds the mapping between IP + and hostnames that will be injected as an entry + in the pod's hosts file. properties: hostnames: description: Hostnames for the above IP address. @@ -2277,55 +3880,123 @@ spec: type: object type: array hostIPC: - description: 'Use the host''s ipc namespace. Optional: Default to false.' + description: 'Use the host''s ipc namespace. Optional: + Default to false.' type: boolean hostNetwork: - description: Host networking requested for this pod. Use the host's network namespace. If this option is set, the ports that will be used must be specified. Default to false. + description: Host networking requested for this pod. + Use the host's network namespace. If this option is + set, the ports that will be used must be specified. + Default to false. type: boolean hostPID: - description: 'Use the host''s pid namespace. Optional: Default to false.' + description: 'Use the host''s pid namespace. Optional: + Default to false.' type: boolean hostname: - description: Specifies the hostname of the Pod If not specified, the pod's hostname will be set to a system-defined value. + description: Specifies the hostname of the Pod If not + specified, the pod's hostname will be set to a system-defined + value. type: string imagePullSecrets: - description: 'ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. If specified, these secrets will be passed to individual puller implementations for them to use. For example, in the case of docker, only DockerConfig type secrets are honored. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + description: 'ImagePullSecrets is an optional list of + references to secrets in the same namespace to use + for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual + puller implementations for them to use. For example, + in the case of docker, only DockerConfig type secrets + are honored. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + description: LocalObjectReference contains enough + information to let you locate the referenced object + inside the same namespace. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' type: string type: object type: array initContainers: - description: 'List of initialization containers belonging to the pod. Init containers are executed in order prior to containers being started. If any init container fails, the pod is considered to have failed and is handled according to its restartPolicy. The name for an init container or normal container must be unique among all containers. Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. The resourceRequirements of an init container are taken into account during scheduling by finding the highest request/limit for each resource type, and then using the max of of that value or the sum of the normal containers. Limits are applied to init containers in a similar fashion. Init containers cannot currently be added or removed. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' + description: 'List of initialization containers belonging + to the pod. Init containers are executed in order + prior to containers being started. If any init container + fails, the pod is considered to have failed and is + handled according to its restartPolicy. The name for + an init container or normal container must be unique + among all containers. Init containers may not have + Lifecycle actions, Readiness probes, Liveness probes, + or Startup probes. The resourceRequirements of an + init container are taken into account during scheduling + by finding the highest request/limit for each resource + type, and then using the max of of that value or the + sum of the normal containers. Limits are applied to + init containers in a similar fashion. Init containers + cannot currently be added or removed. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' items: - description: A single application container that you want to run within a pod. + description: A single application container that you + want to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The + docker image''s CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable + cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within + a shell. The docker image''s ENTRYPOINT is used + if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a + double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to + set in the container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment + variable present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) + are expanded using the previous defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. The $(VAR_NAME) syntax can + be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment + variable's value. Cannot be used if value + is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -2334,56 +4005,84 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the + ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to + select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required + for volumes, optional for env + vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output + format of the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: resource + to select' type: string required: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret + in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret + to select from. Must be a valid + secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the + Secret or its key must be defined type: boolean required: - key @@ -2394,72 +4093,127 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source + of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap + must be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret + must be defined type: boolean type: object type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level + config management to default or override container + images in workload controllers like Deployments + and StatefulSets.' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. + description: Actions that the management system + should take in response to container lifecycle + events. Cannot be updated. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -2467,64 +4221,114 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The reason for termination is passed to + the handler. The Pod''s termination grace + period countdown begins before the PreStop + hooked is executed. Regardless of the outcome + of the handler, the container will eventually + terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until + the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -2532,31 +4336,44 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -2564,31 +4381,53 @@ spec: type: object type: object livenessProbe: - description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2602,76 +4441,121 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + description: Name of the container specified as + a DNS_LABEL. Each container in a pod must have + a unique name (DNS_LABEL). Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. + description: List of ports to expose from the + container. Exposing a port here gives the system + additional information about the network connections + a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent + that port from being exposed. Any port which + is listening on the default "0.0.0.0" address + inside a container will be accessible from the + network. Cannot be updated. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network + port in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external + port to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort @@ -2682,31 +4566,54 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container service + readiness. Container will be removed from service + endpoints if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2720,54 +4627,77 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Compute Resources required by this + container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -2776,7 +4706,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -2785,107 +4717,213 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'Security options the pod should + run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a + read-only root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label + that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label + that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label + that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label + that applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. type: string type: object type: object startupProbe: - description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'StartupProbe indicates that the + Pod has successfully initialized. If specified, + no other probes are executed until this completes + successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. + This can be used to provide different probe + parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data + or warm a cache, than during steady-state operation. + This cannot be updated. This is a beta feature + enabled by the StartupProbe feature flag. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2899,77 +4937,139 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false type: boolean terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block + devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping + of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside + of the container that the device will + be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of + a persistentVolumeClaim in the pod type: string required: - devicePath @@ -2977,27 +5077,46 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting + of a Volume within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. type: string name: - description: This must match the Name of a Volume. + description: This must match the Name of + a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. type: string required: - mountPath @@ -3005,19 +5124,28 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. type: string required: - name type: object type: array nodeName: - description: NodeName is a request to schedule this pod onto a specific node. If it is non-empty, the scheduler simply schedules this pod onto that node, assuming that it fits resource requirements. + description: NodeName is a request to schedule this + pod onto a specific node. If it is non-empty, the + scheduler simply schedules this pod onto that node, + assuming that it fits resource requirements. type: string nodeSelector: additionalProperties: type: string - description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + description: 'NodeSelector is a selector which must + be true for the pod to fit on a node. Selector which + must match a node''s labels for the pod to be scheduled + on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' type: object overhead: additionalProperties: @@ -3026,86 +5154,185 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. This field will be autopopulated at admission time by the RuntimeClass admission controller. If the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. The RuntimeClass admission controller will reject Pod create requests which have the overhead already set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md This field is alpha-level as of Kubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.' + description: 'Overhead represents the resource overhead + associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time + by the RuntimeClass admission controller. If the RuntimeClass + admission controller is enabled, overhead must not + be set in Pod create requests. The RuntimeClass admission + controller will reject Pod create requests which have + the overhead already set. If RuntimeClass is configured + and selected in the PodSpec, Overhead will be set + to the value defined in the corresponding RuntimeClass, + otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md + This field is alpha-level as of Kubernetes v1.16, + and is only honored by servers that enable the PodOverhead + feature.' type: object preemptionPolicy: - description: PreemptionPolicy is the Policy for preempting pods with lower priority. One of Never, PreemptLowerPriority. Defaults to PreemptLowerPriority if unset. This field is alpha-level and is only honored by servers that enable the NonPreemptingPriority feature. + description: PreemptionPolicy is the Policy for preempting + pods with lower priority. One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. This field + is alpha-level and is only honored by servers that + enable the NonPreemptingPriority feature. type: string priority: - description: The priority value. Various system components use this field to find the priority of the pod. When Priority Admission Controller is enabled, it prevents users from setting this field. The admission controller populates this field from PriorityClassName. The higher the value, the higher the priority. + description: The priority value. Various system components + use this field to find the priority of the pod. When + Priority Admission Controller is enabled, it prevents + users from setting this field. The admission controller + populates this field from PriorityClassName. The higher + the value, the higher the priority. format: int32 type: integer priorityClassName: - description: If specified, indicates the pod's priority. "system-node-critical" and "system-cluster-critical" are two special keywords which indicate the highest priorities with the former being the highest priority. Any other name must be defined by creating a PriorityClass object with that name. If not specified, the pod priority will be default or zero if there is no default. + description: If specified, indicates the pod's priority. + "system-node-critical" and "system-cluster-critical" + are two special keywords which indicate the highest + priorities with the former being the highest priority. + Any other name must be defined by creating a PriorityClass + object with that name. If not specified, the pod priority + will be default or zero if there is no default. type: string readinessGates: - description: 'If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to "True" More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md' + description: 'If specified, all readiness gates will + be evaluated for pod readiness. A pod is ready when + all its containers are ready AND all conditions specified + in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md' items: - description: PodReadinessGate contains the reference to a pod condition + description: PodReadinessGate contains the reference + to a pod condition properties: conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. + description: ConditionType refers to a condition + in the pod's condition list with matching type. type: string required: - conditionType type: object type: array restartPolicy: - description: 'Restart policy for all containers within the pod. One of Always, OnFailure, Never. Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' + description: 'Restart policy for all containers within + the pod. One of Always, OnFailure, Never. Default + to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' type: string runtimeClassName: - description: 'RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md This is a beta feature as of Kubernetes v1.14.' + description: 'RuntimeClassName refers to a RuntimeClass + object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches + the named class, the pod will not be run. If unset + or empty, the "legacy" RuntimeClass will be used, + which is an implicit class with an empty definition + that uses the default runtime handler. More info: + https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md + This is a beta feature as of Kubernetes v1.14.' type: string schedulerName: - description: If specified, the pod will be dispatched by specified scheduler. If not specified, the pod will be dispatched by default scheduler. + description: If specified, the pod will be dispatched + by specified scheduler. If not specified, the pod + will be dispatched by default scheduler. type: string securityContext: - description: 'SecurityContext holds pod-level security attributes and common container settings. Optional: Defaults to empty. See type description for default values of each field.' + description: 'SecurityContext holds pod-level security + attributes and common container settings. Optional: + Defaults to empty. See type description for default + values of each field.' properties: fsGroup: - description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume." + description: "A special supplemental group that + applies to all containers in a pod. Some volume + types allow the Kubelet to change the ownership + of that volume to be owned by the pod: \n 1. The + owning GID will be the FSGroup 2. The setgid bit + is set (new files created in the volume will be + owned by FSGroup) 3. The permission bits are OR'd + with rw-rw---- \n If unset, the Kubelet will not + modify the ownership and permissions of any volume." format: int64 type: integer fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified defaults to "Always".' + description: 'fsGroupChangePolicy defines behavior + of changing ownership and permission of the volume + before being exposed inside Pod. This field will + only apply to volume types which support fsGroup + based ownership(and permissions). It will have + no effect on ephemeral volume types such as: secret, + configmaps and emptydir. Valid values are "OnRootMismatch" + and "Always". If not specified defaults to "Always".' type: string runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence + for that container. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will + validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no + such validation will be performed. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + description: The UID to run the entrypoint of the + container process. Defaults to user specified + in image metadata if unspecified. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence for that container. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + description: The SELinux context to be applied to + all containers. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence + for that container. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that + applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that + applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that + applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that + applies to the container. type: string type: object supplementalGroups: - description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container. + description: A list of groups applied to the first + process run in each container, in addition to + the container's primary GID. If unspecified, + no groups will be added to any container. items: format: int64 type: integer type: array sysctls: - description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. + description: Sysctls hold a list of namespaced sysctls + used for the pod. Pods with unsupported sysctls + (by the container runtime) might fail to launch. items: - description: Sysctl defines a kernel parameter to be set + description: Sysctl defines a kernel parameter + to be set properties: name: description: Name of a property to set @@ -3119,79 +5346,161 @@ spec: type: object type: array windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings applied + to all containers. If unspecified, the options + within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where the + GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to run + the entrypoint of the container process. Defaults + to the user specified in image metadata if + unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: string type: object type: object serviceAccount: - description: 'DeprecatedServiceAccount is a depreciated alias for ServiceAccountName. Deprecated: Use serviceAccountName instead.' + description: 'DeprecatedServiceAccount is a depreciated + alias for ServiceAccountName. Deprecated: Use serviceAccountName + instead.' type: string serviceAccountName: - description: 'ServiceAccountName is the name of the ServiceAccount to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + description: 'ServiceAccountName is the name of the + ServiceAccount to use to run this pod. More info: + https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' type: string shareProcessNamespace: - description: 'Share a single process namespace between all of the containers in a pod. When this is set containers will be able to view and signal processes from other containers in the same pod, and the first process in each container will not be assigned PID 1. HostPID and ShareProcessNamespace cannot both be set. Optional: Default to false.' + description: 'Share a single process namespace between + all of the containers in a pod. When this is set containers + will be able to view and signal processes from other + containers in the same pod, and the first process + in each container will not be assigned PID 1. HostPID + and ShareProcessNamespace cannot both be set. Optional: + Default to false.' type: boolean subdomain: - description: If specified, the fully qualified Pod hostname will be "...svc.". If not specified, the pod will not have a domainname at all. + description: If specified, the fully qualified Pod hostname + will be "...svc.". If not specified, the pod will not have + a domainname at all. type: string terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds. + description: Optional duration in seconds the pod needs + to terminate gracefully. May be decreased in delete + request. Value must be non-negative integer. The value + zero indicates delete immediately. If this value is + nil, the default grace period will be used instead. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than + the expected cleanup time for your process. Defaults + to 30 seconds. format: int64 type: integer tolerations: description: If specified, the pod's tolerations. items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + description: The pod this Toleration is attached to + tolerates any taint that matches the triple + using the matching operator . properties: effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + description: Effect indicates the taint effect + to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. type: string key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; + this combination means to match all values and + all keys. type: string operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + description: Operator represents a key's relationship + to the value. Valid operators are Exists and + Equal. Defaults to Equal. Exists is equivalent + to wildcard for value, so that a pod can tolerate + all taints of a particular category. type: string tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + description: TolerationSeconds represents the + period of time the toleration (which must be + of effect NoExecute, otherwise this field is + ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. format: int64 type: integer value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. type: string type: object type: array topologySpreadConstraints: - description: TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. This field is only honored by clusters that enable the EvenPodsSpread feature. All topologySpreadConstraints are ANDed. + description: TopologySpreadConstraints describes how + a group of pods ought to spread across topology domains. + Scheduler will schedule pods in a way which abides + by the constraints. This field is only honored by + clusters that enable the EvenPodsSpread feature. All + topologySpreadConstraints are ANDed. items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. properties: labelSelector: - description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. + description: LabelSelector is used to find matching + pods. Pods that match this label selector are + counted to determine the number of pods in their + corresponding topology domain. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -3203,18 +5512,58 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. type: object type: object maxSkew: - description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.' + description: 'MaxSkew describes the degree to + which pods may be unevenly distributed. It''s + the maximum permitted difference between the + number of matching pods in any two topology + domains of a given topology type. For example, + in a 3-zone cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread as 1/1/0: + | zone1 | zone2 | zone3 | | P | P | | + - if MaxSkew is 1, incoming pod can only be + scheduled to zone3 to become 1/1/1; scheduling + it onto zone1(zone2) would make the ActualSkew(2-0) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew + is 2, incoming pod can be scheduled onto any + zone. It''s a required field. Default value + is 1 and 0 is not allowed.' format: int32 type: integer topologyKey: - description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. It's a required field. + description: TopologyKey is the key of node labels. + Nodes that have a label with this key and identical + values are considered to be in the same topology. + We consider each as a "bucket", + and try to put balanced number of pods into + each bucket. It's a required field. type: string whenUnsatisfiable: - description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as "Unsatisfiable" if and only if placing incoming pod on any topology violates "MaxSkew". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.' + description: 'WhenUnsatisfiable indicates how + to deal with a pod if it doesn''t satisfy the + spread constraint. - DoNotSchedule (default) + tells the scheduler not to schedule it - ScheduleAnyway + tells the scheduler to still schedule it It''s + considered as "Unsatisfiable" if and only if + placing incoming pod on any topology violates + "MaxSkew". For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming pod can only + be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can + still be imbalanced, but scheduler won''t make + it *more* imbalanced. It''s a required field.' type: string required: - maxSkew @@ -3227,62 +5576,105 @@ spec: - whenUnsatisfiable x-kubernetes-list-type: map volumes: - description: 'List of volumes that can be mounted by containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' + description: 'List of volumes that can be mounted by + containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' items: - description: Volume represents a named volume in a pod that may be accessed by any container in the pod. + description: Volume represents a named volume in a + pod that may be accessed by any container in the + pod. properties: awsElasticBlockStore: - description: 'AWSElasticBlockStore represents an AWS Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + description: 'AWSElasticBlockStore represents + an AWS Disk resource that is attached to a kubelet''s + host machine and then exposed to the pod. More + info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' properties: fsType: - description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore TODO: how do we prevent errors in the filesystem from compromising the machine' + description: 'Filesystem type of the volume + that you want to mount. Tip: Ensure that + the filesystem type is supported by the + host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem + from compromising the machine' type: string partition: - description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty).' + description: 'The partition in the volume + that you want to mount. If omitted, the + default is to mount by volume name. Examples: + For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the + property empty).' format: int32 type: integer readOnly: - description: 'Specify "true" to force and set the ReadOnly property in VolumeMounts to "true". If omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + description: 'Specify "true" to force and + set the ReadOnly property in VolumeMounts + to "true". If omitted, the default is "false". + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: boolean volumeID: - description: 'Unique ID of the persistent disk resource in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + description: 'Unique ID of the persistent + disk resource in AWS (Amazon EBS volume). + More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' type: string required: - volumeID type: object azureDisk: - description: AzureDisk represents an Azure Data Disk mount on the host and bind mount to the pod. + description: AzureDisk represents an Azure Data + Disk mount on the host and bind mount to the + pod. properties: cachingMode: - description: 'Host Caching mode: None, Read Only, Read Write.' + description: 'Host Caching mode: None, Read + Only, Read Write.' type: string diskName: - description: The Name of the data disk in the blob storage + description: The Name of the data disk in + the blob storage type: string diskURI: - description: The URI the data disk in the blob storage + description: The URI the data disk in the + blob storage type: string fsType: - description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + description: Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. type: string kind: - description: 'Expected values Shared: multiple blob disks per storage account Dedicated: single blob disk per storage account Managed: azure managed data disk (only in managed availability set). defaults to shared' + description: 'Expected values Shared: multiple + blob disks per storage account Dedicated: + single blob disk per storage account Managed: + azure managed data disk (only in managed + availability set). defaults to shared' type: string readOnly: - description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. + description: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. type: boolean required: - diskName - diskURI type: object azureFile: - description: AzureFile represents an Azure File Service mount on the host and bind mount to the pod. + description: AzureFile represents an Azure File + Service mount on the host and bind mount to + the pod. properties: readOnly: - description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. + description: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. type: boolean secretName: - description: the name of secret that contains Azure Storage Account Name and Key + description: the name of secret that contains + Azure Storage Account Name and Key type: string shareName: description: Share Name @@ -3292,78 +5684,139 @@ spec: - shareName type: object cephfs: - description: CephFS represents a Ceph FS mount on the host that shares a pod's lifetime + description: CephFS represents a Ceph FS mount + on the host that shares a pod's lifetime properties: monitors: - description: 'Required: Monitors is a collection of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'Required: Monitors is a collection + of Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' items: type: string type: array path: - description: 'Optional: Used as the mounted root, rather than the full Ceph tree, default is /' + description: 'Optional: Used as the mounted + root, rather than the full Ceph tree, default + is /' type: string readOnly: - description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'Optional: Defaults to false + (read/write). ReadOnly here will force the + ReadOnly setting in VolumeMounts. More info: + https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: boolean secretFile: - description: 'Optional: SecretFile is the path to key ring for User, default is /etc/ceph/user.secret More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'Optional: SecretFile is the + path to key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string secretRef: - description: 'Optional: SecretRef is reference to the authentication secret for User, default is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'Optional: SecretRef is reference + to the authentication secret for User, default + is empty. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object user: - description: 'Optional: User is the rados user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + description: 'Optional: User is the rados + user name, default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' type: string required: - monitors type: object cinder: - description: 'Cinder represents a cinder volume attached and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + description: 'Cinder represents a cinder volume + attached and mounted on kubelets host machine. + More info: https://examples.k8s.io/mysql-cinder-pd/README.md' properties: fsType: - description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + description: 'Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Examples: "ext4", "xfs", + "ntfs". Implicitly inferred to be "ext4" + if unspecified. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string readOnly: - description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + description: 'Optional: Defaults to false + (read/write). ReadOnly here will force the + ReadOnly setting in VolumeMounts. More info: + https://examples.k8s.io/mysql-cinder-pd/README.md' type: boolean secretRef: - description: 'Optional: points to a secret object containing parameters used to connect to OpenStack.' + description: 'Optional: points to a secret + object containing parameters used to connect + to OpenStack.' properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object volumeID: - description: 'volume id used to identify the volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + description: 'volume id used to identify the + volume in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' type: string required: - volumeID type: object configMap: - description: ConfigMap represents a configMap that should populate this volume + description: ConfigMap represents a configMap + that should populate this volume properties: defaultMode: - description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode bits to use on + created files by default. Must be a value + between 0 and 0777. Defaults to 0644. Directories + within the path are not affected by this + setting. This might be in conflict with + other options that affect the file mode, + like fsGroup, and the result can be other + mode bits set.' format: int32 type: integer items: - description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. + description: Maps a string key to a path + within a volume. properties: key: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode bits to + use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' format: int32 type: integer path: - description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - key @@ -3371,85 +5824,149 @@ spec: type: object type: array name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its keys must be defined + description: Specify whether the ConfigMap + or its keys must be defined type: boolean type: object csi: - description: CSI (Container Storage Interface) represents storage that is handled by an external CSI driver (Alpha feature). + description: CSI (Container Storage Interface) + represents storage that is handled by an external + CSI driver (Alpha feature). properties: driver: - description: Driver is the name of the CSI driver that handles this volume. Consult with your admin for the correct name as registered in the cluster. + description: Driver is the name of the CSI + driver that handles this volume. Consult + with your admin for the correct name as + registered in the cluster. type: string fsType: - description: Filesystem type to mount. Ex. "ext4", "xfs", "ntfs". If not provided, the empty value is passed to the associated CSI driver which will determine the default filesystem to apply. + description: Filesystem type to mount. Ex. + "ext4", "xfs", "ntfs". If not provided, + the empty value is passed to the associated + CSI driver which will determine the default + filesystem to apply. type: string nodePublishSecretRef: - description: NodePublishSecretRef is a reference to the secret object containing sensitive information to pass to the CSI driver to complete the CSI NodePublishVolume and NodeUnpublishVolume calls. This field is optional, and may be empty if no secret is required. If the secret object contains more than one secret, all secret references are passed. + description: NodePublishSecretRef is a reference + to the secret object containing sensitive + information to pass to the CSI driver to + complete the CSI NodePublishVolume and NodeUnpublishVolume + calls. This field is optional, and may + be empty if no secret is required. If the + secret object contains more than one secret, + all secret references are passed. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object readOnly: - description: Specifies a read-only configuration for the volume. Defaults to false (read/write). + description: Specifies a read-only configuration + for the volume. Defaults to false (read/write). type: boolean volumeAttributes: additionalProperties: type: string - description: VolumeAttributes stores driver-specific properties that are passed to the CSI driver. Consult your driver's documentation for supported values. + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. + Consult your driver's documentation for + supported values. type: object required: - driver type: object downwardAPI: - description: DownwardAPI represents downward API about the pod that should populate this volume + description: DownwardAPI represents downward API + about the pod that should populate this volume properties: defaultMode: - description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode bits to use on + created files by default. Must be a value + between 0 and 0777. Defaults to 0644. Directories + within the path are not affected by this + setting. This might be in conflict with + other options that affect the file mode, + like fsGroup, and the result can be other + mode bits set.' format: int32 type: integer items: - description: Items is a list of downward API volume file + description: Items is a list of downward API + volume file items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field properties: fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' + description: 'Required: Selects a field + of the pod: only annotations, labels, + name and namespace are supported.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to + select in the specified API version. type: string required: - fieldPath type: object mode: - description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode bits to + use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' format: int32 type: integer path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute or + contain the ''..'' path. Must be utf-8 + encoded. The first item of the relative + path must not start with ''..''' type: string resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required + for volumes, optional for env + vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output + format of the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: resource + to select' type: string required: - resource @@ -3460,184 +5977,315 @@ spec: type: array type: object emptyDir: - description: 'EmptyDir represents a temporary directory that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + description: 'EmptyDir represents a temporary + directory that shares a pod''s lifetime. More + info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' properties: medium: - description: 'What type of storage medium should back this directory. The default is "" which means to use the node''s default medium. Must be an empty string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + description: 'What type of storage medium + should back this directory. The default + is "" which means to use the node''s default + medium. Must be an empty string (default) + or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' type: string sizeLimit: anyOf: - type: integer - type: string - description: 'Total amount of local storage required for this EmptyDir volume. The size limit is also applicable for memory medium. The maximum usage on memory medium EmptyDir would be the minimum value between the SizeLimit specified here and the sum of memory limits of all containers in a pod. The default is nil which means that the limit is undefined. More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + description: 'Total amount of local storage + required for this EmptyDir volume. The size + limit is also applicable for memory medium. + The maximum usage on memory medium EmptyDir + would be the minimum value between the SizeLimit + specified here and the sum of memory limits + of all containers in a pod. The default + is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: object fc: - description: FC represents a Fibre Channel resource that is attached to a kubelet's host machine and then exposed to the pod. + description: FC represents a Fibre Channel resource + that is attached to a kubelet's host machine + and then exposed to the pod. properties: fsType: - description: 'Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. TODO: how do we prevent errors in the filesystem from compromising the machine' + description: 'Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. + TODO: how do we prevent errors in the filesystem + from compromising the machine' type: string lun: description: 'Optional: FC target lun number' format: int32 type: integer readOnly: - description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' + description: 'Optional: Defaults to false + (read/write). ReadOnly here will force the + ReadOnly setting in VolumeMounts.' type: boolean targetWWNs: - description: 'Optional: FC target worldwide names (WWNs)' + description: 'Optional: FC target worldwide + names (WWNs)' items: type: string type: array wwids: - description: 'Optional: FC volume world wide identifiers (wwids) Either wwids or combination of targetWWNs and lun must be set, but not both simultaneously.' + description: 'Optional: FC volume world wide + identifiers (wwids) Either wwids or combination + of targetWWNs and lun must be set, but not + both simultaneously.' items: type: string type: array type: object flexVolume: - description: FlexVolume represents a generic volume resource that is provisioned/attached using an exec based plugin. + description: FlexVolume represents a generic volume + resource that is provisioned/attached using + an exec based plugin. properties: driver: - description: Driver is the name of the driver to use for this volume. + description: Driver is the name of the driver + to use for this volume. type: string fsType: - description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". The default filesystem depends on FlexVolume script. + description: Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". + The default filesystem depends on FlexVolume + script. type: string options: additionalProperties: type: string - description: 'Optional: Extra command options if any.' + description: 'Optional: Extra command options + if any.' type: object readOnly: - description: 'Optional: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts.' + description: 'Optional: Defaults to false + (read/write). ReadOnly here will force the + ReadOnly setting in VolumeMounts.' type: boolean secretRef: - description: 'Optional: SecretRef is reference to the secret object containing sensitive information to pass to the plugin scripts. This may be empty if no secret object is specified. If the secret object contains more than one secret, all secrets are passed to the plugin scripts.' + description: 'Optional: SecretRef is reference + to the secret object containing sensitive + information to pass to the plugin scripts. + This may be empty if no secret object is + specified. If the secret object contains + more than one secret, all secrets are passed + to the plugin scripts.' properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object required: - driver type: object flocker: - description: Flocker represents a Flocker volume attached to a kubelet's host machine. This depends on the Flocker control service being running + description: Flocker represents a Flocker volume + attached to a kubelet's host machine. This depends + on the Flocker control service being running properties: datasetName: - description: Name of the dataset stored as metadata -> name on the dataset for Flocker should be considered as deprecated + description: Name of the dataset stored as + metadata -> name on the dataset for Flocker + should be considered as deprecated type: string datasetUUID: - description: UUID of the dataset. This is unique identifier of a Flocker dataset + description: UUID of the dataset. This is + unique identifier of a Flocker dataset type: string type: object gcePersistentDisk: - description: 'GCEPersistentDisk represents a GCE Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + description: 'GCEPersistentDisk represents a GCE + Disk resource that is attached to a kubelet''s + host machine and then exposed to the pod. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' properties: fsType: - description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk TODO: how do we prevent errors in the filesystem from compromising the machine' + description: 'Filesystem type of the volume + that you want to mount. Tip: Ensure that + the filesystem type is supported by the + host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem + from compromising the machine' type: string partition: - description: 'The partition in the volume that you want to mount. If omitted, the default is to mount by volume name. Examples: For volume /dev/sda1, you specify the partition as "1". Similarly, the volume partition for /dev/sda is "0" (or you can leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + description: 'The partition in the volume + that you want to mount. If omitted, the + default is to mount by volume name. Examples: + For volume /dev/sda1, you specify the partition + as "1". Similarly, the volume partition + for /dev/sda is "0" (or you can leave the + property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' format: int32 type: integer pdName: - description: 'Unique name of the PD resource in GCE. Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + description: 'Unique name of the PD resource + in GCE. Used to identify the disk in GCE. + More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: string readOnly: - description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + description: 'ReadOnly here will force the + ReadOnly setting in VolumeMounts. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' type: boolean required: - pdName type: object gitRepo: - description: 'GitRepo represents a git repository at a particular revision. DEPRECATED: GitRepo is deprecated. To provision a container with a git repo, mount an EmptyDir into an InitContainer that clones the repo using git, then mount the EmptyDir into the Pod''s container.' + description: 'GitRepo represents a git repository + at a particular revision. DEPRECATED: GitRepo + is deprecated. To provision a container with + a git repo, mount an EmptyDir into an InitContainer + that clones the repo using git, then mount the + EmptyDir into the Pod''s container.' properties: directory: - description: Target directory name. Must not contain or start with '..'. If '.' is supplied, the volume directory will be the git repository. Otherwise, if specified, the volume will contain the git repository in the subdirectory with the given name. + description: Target directory name. Must not + contain or start with '..'. If '.' is supplied, + the volume directory will be the git repository. Otherwise, + if specified, the volume will contain the + git repository in the subdirectory with + the given name. type: string repository: description: Repository URL type: string revision: - description: Commit hash for the specified revision. + description: Commit hash for the specified + revision. type: string required: - repository type: object glusterfs: - description: 'Glusterfs represents a Glusterfs mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + description: 'Glusterfs represents a Glusterfs + mount on the host that shares a pod''s lifetime. + More info: https://examples.k8s.io/volumes/glusterfs/README.md' properties: endpoints: - description: 'EndpointsName is the endpoint name that details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + description: 'EndpointsName is the endpoint + name that details Glusterfs topology. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string path: - description: 'Path is the Glusterfs volume path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + description: 'Path is the Glusterfs volume + path. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: string readOnly: - description: 'ReadOnly here will force the Glusterfs volume to be mounted with read-only permissions. Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + description: 'ReadOnly here will force the + Glusterfs volume to be mounted with read-only + permissions. Defaults to false. More info: + https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' type: boolean required: - endpoints - path type: object hostPath: - description: 'HostPath represents a pre-existing file or directory on the host machine that is directly exposed to the container. This is generally used for system agents or other privileged things that are allowed to see the host machine. Most containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath --- TODO(jonesdl) We need to restrict who can use host directory mounts and who can/can not mount host directories as read/write.' + description: 'HostPath represents a pre-existing + file or directory on the host machine that is + directly exposed to the container. This is generally + used for system agents or other privileged things + that are allowed to see the host machine. Most + containers will NOT need this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can + use host directory mounts and who can/can not + mount host directories as read/write.' properties: path: - description: 'Path of the directory on the host. If the path is a symlink, it will follow the link to the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + description: 'Path of the directory on the + host. If the path is a symlink, it will + follow the link to the real path. More info: + https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string type: - description: 'Type for HostPath Volume Defaults to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + description: 'Type for HostPath Volume Defaults + to "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' type: string required: - path type: object iscsi: - description: 'ISCSI represents an ISCSI Disk resource that is attached to a kubelet''s host machine and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + description: 'ISCSI represents an ISCSI Disk resource + that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' properties: chapAuthDiscovery: - description: whether support iSCSI Discovery CHAP authentication + description: whether support iSCSI Discovery + CHAP authentication type: boolean chapAuthSession: - description: whether support iSCSI Session CHAP authentication + description: whether support iSCSI Session + CHAP authentication type: boolean fsType: - description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi TODO: how do we prevent errors in the filesystem from compromising the machine' + description: 'Filesystem type of the volume + that you want to mount. Tip: Ensure that + the filesystem type is supported by the + host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem + from compromising the machine' type: string initiatorName: - description: Custom iSCSI Initiator Name. If initiatorName is specified with iscsiInterface simultaneously, new iSCSI interface : will be created for the connection. + description: Custom iSCSI Initiator Name. + If initiatorName is specified with iscsiInterface + simultaneously, new iSCSI interface : will be created for + the connection. type: string iqn: description: Target iSCSI Qualified Name. type: string iscsiInterface: - description: iSCSI Interface Name that uses an iSCSI transport. Defaults to 'default' (tcp). + description: iSCSI Interface Name that uses + an iSCSI transport. Defaults to 'default' + (tcp). type: string lun: description: iSCSI Target Lun number. format: int32 type: integer portals: - description: iSCSI Target Portal List. The portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). + description: iSCSI Target Portal List. The + portal is either an IP or ip_addr:port if + the port is other than default (typically + TCP ports 860 and 3260). items: type: string type: array readOnly: - description: ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. + description: ReadOnly here will force the + ReadOnly setting in VolumeMounts. Defaults + to false. type: boolean secretRef: - description: CHAP Secret for iSCSI target and initiator authentication + description: CHAP Secret for iSCSI target + and initiator authentication properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object targetPortal: - description: iSCSI Target Portal. The Portal is either an IP or ip_addr:port if the port is other than default (typically TCP ports 860 and 3260). + description: iSCSI Target Portal. The Portal + is either an IP or ip_addr:port if the port + is other than default (typically TCP ports + 860 and 3260). type: string required: - iqn @@ -3645,92 +6293,159 @@ spec: - targetPortal type: object name: - description: 'Volume''s name. Must be a DNS_LABEL and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: 'Volume''s name. Must be a DNS_LABEL + and unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string nfs: - description: 'NFS represents an NFS mount on the host that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + description: 'NFS represents an NFS mount on the + host that shares a pod''s lifetime More info: + https://kubernetes.io/docs/concepts/storage/volumes#nfs' properties: path: - description: 'Path that is exported by the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + description: 'Path that is exported by the + NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string readOnly: - description: 'ReadOnly here will force the NFS export to be mounted with read-only permissions. Defaults to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + description: 'ReadOnly here will force the + NFS export to be mounted with read-only + permissions. Defaults to false. More info: + https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: boolean server: - description: 'Server is the hostname or IP address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + description: 'Server is the hostname or IP + address of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' type: string required: - path - server type: object persistentVolumeClaim: - description: 'PersistentVolumeClaimVolumeSource represents a reference to a PersistentVolumeClaim in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + description: 'PersistentVolumeClaimVolumeSource + represents a reference to a PersistentVolumeClaim + in the same namespace. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' properties: claimName: - description: 'ClaimName is the name of a PersistentVolumeClaim in the same namespace as the pod using this volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this + volume. More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' type: string readOnly: - description: Will force the ReadOnly setting in VolumeMounts. Default false. + description: Will force the ReadOnly setting + in VolumeMounts. Default false. type: boolean required: - claimName type: object photonPersistentDisk: - description: PhotonPersistentDisk represents a PhotonController persistent disk attached and mounted on kubelets host machine + description: PhotonPersistentDisk represents a + PhotonController persistent disk attached and + mounted on kubelets host machine properties: fsType: - description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + description: Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. type: string pdID: - description: ID that identifies Photon Controller persistent disk + description: ID that identifies Photon Controller + persistent disk type: string required: - pdID type: object portworxVolume: - description: PortworxVolume represents a portworx volume attached and mounted on kubelets host machine + description: PortworxVolume represents a portworx + volume attached and mounted on kubelets host + machine properties: fsType: - description: FSType represents the filesystem type to mount Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs". Implicitly inferred to be "ext4" if unspecified. + description: FSType represents the filesystem + type to mount Must be a filesystem type + supported by the host operating system. + Ex. "ext4", "xfs". Implicitly inferred to + be "ext4" if unspecified. type: string readOnly: - description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. + description: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. type: boolean volumeID: - description: VolumeID uniquely identifies a Portworx volume + description: VolumeID uniquely identifies + a Portworx volume type: string required: - volumeID type: object projected: - description: Items for all in one resources secrets, configmaps, and downward API + description: Items for all in one resources secrets, + configmaps, and downward API properties: defaultMode: - description: Mode bits to use on created files by default. Must be a value between 0 and 0777. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set. + description: Mode bits to use on created files + by default. Must be a value between 0 and + 0777. Directories within the path are not + affected by this setting. This might be + in conflict with other options that affect + the file mode, like fsGroup, and the result + can be other mode bits set. format: int32 type: integer sources: description: list of volume projections items: - description: Projection that may be projected along with other supported volume types + description: Projection that may be projected + along with other supported volume types properties: configMap: - description: information about the configMap data to project + description: information about the configMap + data to project properties: items: - description: If unspecified, each key-value pair in the Data field of the referenced ConfigMap will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the ConfigMap, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. + description: If unspecified, each + key-value pair in the Data field + of the referenced ConfigMap will + be projected into the volume as + a file whose name is the key and + content is the value. If specified, + the listed keys will be projected + into the specified paths, and + unlisted keys will not be present. + If a key is specified which is + not present in the ConfigMap, + the volume setup will error unless + it is marked optional. Paths must + be relative and may not contain + the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. + description: Maps a string key + to a path within a volume. properties: key: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode + bits to use on this file, + must be a value between + 0 and 0777. If not specified, + the volume defaultMode will + be used. This might be in + conflict with other options + that affect the file mode, + like fsGroup, and the result + can be other mode bits set.' format: int32 type: integer path: - description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. + description: The relative + path of the file to map + the key to. May not be an + absolute path. May not contain + the path element '..'. May + not start with the string + '..'. type: string required: - key @@ -3738,54 +6453,100 @@ spec: type: object type: array name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its keys must be defined + description: Specify whether the + ConfigMap or its keys must be + defined type: boolean type: object downwardAPI: - description: information about the downwardAPI data to project + description: information about the downwardAPI + data to project properties: items: - description: Items is a list of DownwardAPIVolume file + description: Items is a list of + DownwardAPIVolume file items: - description: DownwardAPIVolumeFile represents information to create the file containing the pod field + description: DownwardAPIVolumeFile + represents information to create + the file containing the pod + field properties: fieldRef: - description: 'Required: Selects a field of the pod: only annotations, labels, name and namespace are supported.' + description: 'Required: Selects + a field of the pod: only + annotations, labels, name + and namespace are supported.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of + the schema the FieldPath + is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the + field to select in the + specified API version. type: string required: - fieldPath type: object mode: - description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode + bits to use on this file, + must be a value between + 0 and 0777. If not specified, + the volume defaultMode will + be used. This might be in + conflict with other options + that affect the file mode, + like fsGroup, and the result + can be other mode bits set.' format: int32 type: integer path: - description: 'Required: Path is the relative path name of the file to be created. Must not be absolute or contain the ''..'' path. Must be utf-8 encoded. The first item of the relative path must not start with ''..''' + description: 'Required: Path + is the relative path name + of the file to be created. + Must not be absolute or + contain the ''..'' path. + Must be utf-8 encoded. The + first item of the relative + path must not start with + ''..''' type: string resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, requests.cpu and requests.memory) are currently supported.' + description: 'Selects a resource + of the container: only resources + limits and requests (limits.cpu, + limits.memory, requests.cpu + and requests.memory) are + currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container + name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies + the output format of + the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: + resource to select' type: string required: - resource @@ -3796,22 +6557,53 @@ spec: type: array type: object secret: - description: information about the secret data to project + description: information about the secret + data to project properties: items: - description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. + description: If unspecified, each + key-value pair in the Data field + of the referenced Secret will + be projected into the volume as + a file whose name is the key and + content is the value. If specified, + the listed keys will be projected + into the specified paths, and + unlisted keys will not be present. + If a key is specified which is + not present in the Secret, the + volume setup will error unless + it is marked optional. Paths must + be relative and may not contain + the '..' path or start with '..'. items: - description: Maps a string key to a path within a volume. + description: Maps a string key + to a path within a volume. properties: key: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode + bits to use on this file, + must be a value between + 0 and 0777. If not specified, + the volume defaultMode will + be used. This might be in + conflict with other options + that affect the file mode, + like fsGroup, and the result + can be other mode bits set.' format: int32 type: integer path: - description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. + description: The relative + path of the file to map + the key to. May not be an + absolute path. May not contain + the path element '..'. May + not start with the string + '..'. type: string required: - key @@ -3819,24 +6611,50 @@ spec: type: object type: array name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the + Secret or its key must be defined type: boolean type: object serviceAccountToken: - description: information about the serviceAccountToken data to project + description: information about the serviceAccountToken + data to project properties: audience: - description: Audience is the intended audience of the token. A recipient of a token must identify itself with an identifier specified in the audience of the token, and otherwise should reject the token. The audience defaults to the identifier of the apiserver. + description: Audience is the intended + audience of the token. A recipient + of a token must identify itself + with an identifier specified in + the audience of the token, and + otherwise should reject the token. + The audience defaults to the identifier + of the apiserver. type: string expirationSeconds: - description: ExpirationSeconds is the requested duration of validity of the service account token. As the token approaches expiration, the kubelet volume plugin will proactively rotate the service account token. The kubelet will start trying to rotate the token if the token is older than 80 percent of its time to live or if the token is older than 24 hours.Defaults to 1 hour and must be at least 10 minutes. + description: ExpirationSeconds is + the requested duration of validity + of the service account token. + As the token approaches expiration, + the kubelet volume plugin will + proactively rotate the service + account token. The kubelet will + start trying to rotate the token + if the token is older than 80 + percent of its time to live or + if the token is older than 24 + hours.Defaults to 1 hour and must + be at least 10 minutes. format: int64 type: integer path: - description: Path is the path relative to the mount point of the file to project the token into. + description: Path is the path relative + to the mount point of the file + to project the token into. type: string required: - path @@ -3847,103 +6665,159 @@ spec: - sources type: object quobyte: - description: Quobyte represents a Quobyte mount on the host that shares a pod's lifetime + description: Quobyte represents a Quobyte mount + on the host that shares a pod's lifetime properties: group: - description: Group to map volume access to Default is no group + description: Group to map volume access to + Default is no group type: string readOnly: - description: ReadOnly here will force the Quobyte volume to be mounted with read-only permissions. Defaults to false. + description: ReadOnly here will force the + Quobyte volume to be mounted with read-only + permissions. Defaults to false. type: boolean registry: - description: Registry represents a single or multiple Quobyte Registry services specified as a string as host:port pair (multiple entries are separated with commas) which acts as the central registry for volumes + description: Registry represents a single + or multiple Quobyte Registry services specified + as a string as host:port pair (multiple + entries are separated with commas) which + acts as the central registry for volumes type: string tenant: - description: Tenant owning the given Quobyte volume in the Backend Used with dynamically provisioned Quobyte volumes, value is set by the plugin + description: Tenant owning the given Quobyte + volume in the Backend Used with dynamically + provisioned Quobyte volumes, value is set + by the plugin type: string user: - description: User to map volume access to Defaults to serivceaccount user + description: User to map volume access to + Defaults to serivceaccount user type: string volume: - description: Volume is a string that references an already created Quobyte volume by name. + description: Volume is a string that references + an already created Quobyte volume by name. type: string required: - registry - volume type: object rbd: - description: 'RBD represents a Rados Block Device mount on the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/rbd/README.md' + description: 'RBD represents a Rados Block Device + mount on the host that shares a pod''s lifetime. + More info: https://examples.k8s.io/volumes/rbd/README.md' properties: fsType: - description: 'Filesystem type of the volume that you want to mount. Tip: Ensure that the filesystem type is supported by the host operating system. Examples: "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd TODO: how do we prevent errors in the filesystem from compromising the machine' + description: 'Filesystem type of the volume + that you want to mount. Tip: Ensure that + the filesystem type is supported by the + host operating system. Examples: "ext4", + "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem + from compromising the machine' type: string image: - description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'The rados image name. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string keyring: - description: 'Keyring is the path to key ring for RBDUser. Default is /etc/ceph/keyring. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'Keyring is the path to key ring + for RBDUser. Default is /etc/ceph/keyring. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string monitors: - description: 'A collection of Ceph monitors. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'A collection of Ceph monitors. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' items: type: string type: array pool: - description: 'The rados pool name. Default is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'The rados pool name. Default + is rbd. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string readOnly: - description: 'ReadOnly here will force the ReadOnly setting in VolumeMounts. Defaults to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'ReadOnly here will force the + ReadOnly setting in VolumeMounts. Defaults + to false. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: boolean secretRef: - description: 'SecretRef is name of the authentication secret for RBDUser. If provided overrides keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides + keyring. Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object user: - description: 'The rados user name. Default is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + description: 'The rados user name. Default + is admin. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' type: string required: - image - monitors type: object scaleIO: - description: ScaleIO represents a ScaleIO persistent volume attached and mounted on Kubernetes nodes. + description: ScaleIO represents a ScaleIO persistent + volume attached and mounted on Kubernetes nodes. properties: fsType: - description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Default is "xfs". + description: Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". + Default is "xfs". type: string gateway: - description: The host address of the ScaleIO API Gateway. + description: The host address of the ScaleIO + API Gateway. type: string protectionDomain: - description: The name of the ScaleIO Protection Domain for the configured storage. + description: The name of the ScaleIO Protection + Domain for the configured storage. type: string readOnly: - description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. + description: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. type: boolean secretRef: - description: SecretRef references to the secret for ScaleIO user and other sensitive information. If this is not provided, Login operation will fail. + description: SecretRef references to the secret + for ScaleIO user and other sensitive information. + If this is not provided, Login operation + will fail. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object sslEnabled: - description: Flag to enable/disable SSL communication with Gateway, default false + description: Flag to enable/disable SSL communication + with Gateway, default false type: boolean storageMode: - description: Indicates whether the storage for a volume should be ThickProvisioned or ThinProvisioned. Default is ThinProvisioned. + description: Indicates whether the storage + for a volume should be ThickProvisioned + or ThinProvisioned. Default is ThinProvisioned. type: string storagePool: - description: The ScaleIO Storage Pool associated with the protection domain. + description: The ScaleIO Storage Pool associated + with the protection domain. type: string system: - description: The name of the storage system as configured in ScaleIO. + description: The name of the storage system + as configured in ScaleIO. type: string volumeName: - description: The name of a volume already created in the ScaleIO system that is associated with this volume source. + description: The name of a volume already + created in the ScaleIO system that is associated + with this volume source. type: string required: - gateway @@ -3951,26 +6825,57 @@ spec: - system type: object secret: - description: 'Secret represents a secret that should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + description: 'Secret represents a secret that + should populate this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' properties: defaultMode: - description: 'Optional: mode bits to use on created files by default. Must be a value between 0 and 0777. Defaults to 0644. Directories within the path are not affected by this setting. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode bits to use on + created files by default. Must be a value + between 0 and 0777. Defaults to 0644. Directories + within the path are not affected by this + setting. This might be in conflict with + other options that affect the file mode, + like fsGroup, and the result can be other + mode bits set.' format: int32 type: integer items: - description: If unspecified, each key-value pair in the Data field of the referenced Secret will be projected into the volume as a file whose name is the key and content is the value. If specified, the listed keys will be projected into the specified paths, and unlisted keys will not be present. If a key is specified which is not present in the Secret, the volume setup will error unless it is marked optional. Paths must be relative and may not contain the '..' path or start with '..'. + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed keys + will be projected into the specified paths, + and unlisted keys will not be present. If + a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. items: - description: Maps a string key to a path within a volume. + description: Maps a string key to a path + within a volume. properties: key: description: The key to project. type: string mode: - description: 'Optional: mode bits to use on this file, must be a value between 0 and 0777. If not specified, the volume defaultMode will be used. This might be in conflict with other options that affect the file mode, like fsGroup, and the result can be other mode bits set.' + description: 'Optional: mode bits to + use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can be + other mode bits set.' format: int32 type: integer path: - description: The relative path of the file to map the key to. May not be an absolute path. May not contain the path element '..'. May not start with the string '..'. + description: The relative path of the + file to map the key to. May not be + an absolute path. May not contain + the path element '..'. May not start + with the string '..'. type: string required: - key @@ -3978,49 +6883,81 @@ spec: type: object type: array optional: - description: Specify whether the Secret or its keys must be defined + description: Specify whether the Secret or + its keys must be defined type: boolean secretName: - description: 'Name of the secret in the pod''s namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + description: 'Name of the secret in the pod''s + namespace to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' type: string type: object storageos: - description: StorageOS represents a StorageOS volume attached and mounted on Kubernetes nodes. + description: StorageOS represents a StorageOS + volume attached and mounted on Kubernetes nodes. properties: fsType: - description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + description: Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. type: string readOnly: - description: Defaults to false (read/write). ReadOnly here will force the ReadOnly setting in VolumeMounts. + description: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting + in VolumeMounts. type: boolean secretRef: - description: SecretRef specifies the secret to use for obtaining the StorageOS API credentials. If not specified, default values will be attempted. + description: SecretRef specifies the secret + to use for obtaining the StorageOS API credentials. If + not specified, default values will be attempted. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string type: object volumeName: - description: VolumeName is the human-readable name of the StorageOS volume. Volume names are only unique within a namespace. + description: VolumeName is the human-readable + name of the StorageOS volume. Volume names + are only unique within a namespace. type: string volumeNamespace: - description: VolumeNamespace specifies the scope of the volume within StorageOS. If no namespace is specified then the Pod's namespace will be used. This allows the Kubernetes name scoping to be mirrored within StorageOS for tighter integration. Set VolumeName to any name to override the default behaviour. Set to "default" if you are not using namespaces within StorageOS. Namespaces that do not pre-exist within StorageOS will be created. + description: VolumeNamespace specifies the + scope of the volume within StorageOS. If + no namespace is specified then the Pod's + namespace will be used. This allows the + Kubernetes name scoping to be mirrored within + StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. + Set to "default" if you are not using namespaces + within StorageOS. Namespaces that do not + pre-exist within StorageOS will be created. type: string type: object vsphereVolume: - description: VsphereVolume represents a vSphere volume attached and mounted on kubelets host machine + description: VsphereVolume represents a vSphere + volume attached and mounted on kubelets host + machine properties: fsType: - description: Filesystem type to mount. Must be a filesystem type supported by the host operating system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred to be "ext4" if unspecified. + description: Filesystem type to mount. Must + be a filesystem type supported by the host + operating system. Ex. "ext4", "xfs", "ntfs". + Implicitly inferred to be "ext4" if unspecified. type: string storagePolicyID: - description: Storage Policy Based Management (SPBM) profile ID associated with the StoragePolicyName. + description: Storage Policy Based Management + (SPBM) profile ID associated with the StoragePolicyName. type: string storagePolicyName: - description: Storage Policy Based Management (SPBM) profile name. + description: Storage Policy Based Management + (SPBM) profile name. type: string volumePath: - description: Path that identifies vSphere volume vmdk + description: Path that identifies vSphere + volume vmdk type: string required: - volumePath @@ -4035,7 +6972,8 @@ spec: type: object type: array engineResources: - description: ResourceRequirements describes the compute resource requirements. + description: ResourceRequirements describes the compute resource + requirements. properties: limits: additionalProperties: @@ -4044,7 +6982,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -4053,7 +6992,10 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object explainer: @@ -4063,31 +7005,60 @@ spec: type: string type: object containerSpec: - description: A single application container that you want to run within a pod. + description: A single application container that you want + to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The docker + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within a + shell. The docker image''s ENTRYPOINT is used if this + is not provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string + will be unchanged. The $(VAR_NAME) syntax can be escaped + with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to set in the + container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable + present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. Must + be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) are + expanded using the previous defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will + never be expanded, regardless of whether the variable + exists or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's + value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -4096,37 +7067,53 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap + or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select + in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for + volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format + of the exposed resources, defaults to + "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -4136,16 +7123,22 @@ spec: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret in the + pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select + from. Must be a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret + or its key must be defined type: boolean required: - key @@ -4156,66 +7149,111 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported + as an event when the container is starting. When a key + exists in multiple sources, the value associated with + the last source will take precedence. Values defined + by an Env with a duplicate key will take precedence. + Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source of + a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap must + be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret must + be defined type: boolean type: object type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images in + workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is specified, + or IfNotPresent otherwise. Cannot be updated. More info: + https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. + description: Actions that the management system should + take in response to container lifecycle events. Cannot + be updated. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according + to its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the action + to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the + request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom + header to be used in HTTP probes properties: name: description: The header field name @@ -4235,52 +7273,90 @@ spec: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup probe + failure, preemption, resource contention, etc. The + handler is not called if the container crashes or + exits. The reason for termination is passed to the + handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will + eventually terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the action + to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the + request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom + header to be used in HTTP probes properties: name: description: The header field name @@ -4300,25 +7376,35 @@ spec: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -4326,31 +7412,49 @@ spec: type: object type: object livenessProbe: - description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following should + be specified. Exec specifies the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request to + perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the request. + HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom header + to be used in HTTP probes properties: name: description: The header field name @@ -4370,70 +7474,107 @@ spec: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting to the + host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. + Minimum value is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect to, + defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. + description: List of ports to expose from the container. + Exposing a port here gives the system additional information + about the network connections a container uses, but + is primarily informational. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port + which is listening on the default "0.0.0.0" address + inside a container will be accessible from the network. + Cannot be updated. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network port + in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external port + to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the port + that can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". type: string required: - containerPort @@ -4444,31 +7585,49 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if + the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following should + be specified. Exec specifies the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request to + perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the request. + HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom header + to be used in HTTP probes properties: name: description: The header field name @@ -4488,48 +7647,65 @@ spec: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting to the + host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. + Minimum value is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect to, + defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -4538,7 +7714,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -4547,107 +7724,193 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'Security options the pod should run with. + More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as + Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a read-only + root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not + run as UID 0 (root) and fail to start the container + if it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint of the + container process. Defaults to user specified in + image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that + applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that + applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that + applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that + applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings applied + to all containers. If unspecified, the options from + the PodSecurityContext will be used. If set in both + SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to run the + entrypoint of the container process. Defaults + to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: string type: object type: object startupProbe: - description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'StartupProbe indicates that the Pod has + successfully initialized. If specified, no other probes + are executed until this completes successfully. If this + probe fails, the Pod will be restarted, just as if the + livenessProbe failed. This can be used to provide different + probe parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data or warm + a cache, than during steady-state operation. This cannot + be updated. This is a beta feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following should + be specified. Exec specifies the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request to + perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the request. + HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom header + to be used in HTTP probes properties: name: description: The header field name @@ -4667,71 +7930,120 @@ spec: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting to the + host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. + Minimum value is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect to, + defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate a + buffer for stdin in the container runtime. If this is + not set, reads from stdin in the container will always + result in EOF. Default is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is + empty until the first client attaches to stdin, and + then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false type: boolean terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such as + an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length + across all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate a + TTY for itself, also requires 'stdin' to be true. Default + is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block devices + to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping of a raw + block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside of the + container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of a persistentVolumeClaim + in the pod type: string required: - devicePath @@ -4739,27 +8051,43 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting of a Volume + within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from which the + container's volume should be mounted. Defaults + to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the + container's environment. Defaults to "" (volume's + root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -4767,13 +8095,22 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot be + updated. type: string required: - name type: object endpoint: properties: + grpc_port: + format: int32 + type: integer + http_port: + format: int32 + type: integer service_host: type: string service_port: @@ -4798,6 +8135,12 @@ spec: type: array endpoint: properties: + grpc_port: + format: int32 + type: integer + http_port: + format: int32 + type: integer service_host: type: string service_port: @@ -4811,7 +8154,9 @@ spec: implementation: type: string logger: - description: Request/response payload logging. v2alpha1 feature that is added to v1 for backwards compatibility while v1 is the storage version. + description: Request/response payload logging. v2alpha1 feature + that is added to v1 for backwards compatibility while v1 + is the storage version. properties: mode: description: What payloads to log @@ -4870,16 +8215,27 @@ spec: properties: env: items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present + in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. Must + be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in + the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to + "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -4888,37 +8244,52 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or + its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in + the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of + the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -4928,16 +8299,22 @@ spec: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret in the pod's + namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select + from. Must be a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its + key must be defined type: boolean required: - key @@ -4951,7 +8328,8 @@ spec: format: int32 type: integer resources: - description: ResourceRequirements describes the compute resource requirements. + description: ResourceRequirements describes the compute resource + requirements. properties: limits: additionalProperties: @@ -4960,7 +8338,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -4969,7 +8348,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object type: object diff --git a/operator/config/manager/configmap.yaml b/operator/config/manager/configmap.yaml index 9de7918203..a439e71cd1 100644 --- a/operator/config/manager/configmap.yaml +++ b/operator/config/manager/configmap.yaml @@ -18,55 +18,59 @@ data: predictor_servers: |- { "TENSORFLOW_SERVER": { - "tensorflow": true, - "tfImage": "tensorflow/serving:2.1.0", - "rest": { - "image": "seldonio/tfserving-proxy_rest", - "defaultImageVersion": "1.5.0-dev" - }, - "grpc": { - "image": "seldonio/tfserving-proxy_grpc", + "protocols" : { + "tensorflow": { + "image": "tensorflow/serving", + "defaultImageVersion": "2.1.0" + }, + "seldon": { + "image": "seldonio/tfserving-proxy", "defaultImageVersion": "1.5.0-dev" + } } }, "SKLEARN_SERVER": { - "rest": { - "image": "seldonio/sklearnserver_rest", - "defaultImageVersion": "1.5.0-dev" - }, - "grpc": { - "image": "seldonio/sklearnserver_grpc", + "protocols" : { + "seldon": { + "image": "seldonio/sklearnserver", "defaultImageVersion": "1.5.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } } }, "XGBOOST_SERVER": { - "rest": { - "image": "seldonio/xgboostserver_rest", - "defaultImageVersion": "1.5.0-dev" - }, - "grpc": { - "image": "seldonio/xgboostserver_grpc", + "protocols" : { + "seldon": { + "image": "seldonio/xgboostserver", "defaultImageVersion": "1.5.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } } }, "MLFLOW_SERVER": { - "rest": { - "image": "seldonio/mlflowserver_rest", - "defaultImageVersion": "1.5.0-dev" - }, - "grpc": { - "image": "seldonio/mlflowserver_grpc", + "protocols" : { + "seldon": { + "image": "seldonio/mlflowserver", "defaultImageVersion": "1.5.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } } }, "TRITON_SERVER": { - "rest": { - "image": "nvcr.io/nvidia/tritonserver", - "defaultImageVersion": "20.08-py3" - }, - "grpc": { + "protocols" : { + "kfserving": { "image": "nvcr.io/nvidia/tritonserver", "defaultImageVersion": "20.08-py3" + } } } } diff --git a/operator/constants/constants.go b/operator/constants/constants.go index ad3200e154..22d7fd4739 100644 --- a/operator/constants/constants.go +++ b/operator/constants/constants.go @@ -10,13 +10,15 @@ const ( PrePackedServerTensorflow = "TENSORFLOW_SERVER" PrePackedServerSklearn = "SKLEARN_SERVER" PrePackedServerTriton = "TRITON_SERVER" + PrePackedMlflow = "MLFLOW_SERVER" TfServingGrpcPort = 2000 TfServingRestPort = 2001 TfServingArgPort = "--port=" TfServingArgRestPort = "--rest_api_port=" - FirstPortNumber = int32(9000) + FirstHttpPortNumber = int32(9000) + FirstGrpcPortNumber = int32(9500) DNSLocalHost = "localhost" DNSClusterLocalSuffix = ".svc.cluster.local." GrpcPortName = "grpc" diff --git a/operator/controllers/ambassador.go b/operator/controllers/ambassador.go index c2279d38b6..8c689af613 100644 --- a/operator/controllers/ambassador.go +++ b/operator/controllers/ambassador.go @@ -432,29 +432,11 @@ func getAmbassadorConfigs(mlDep *machinelearningv1.SeldonDeployment, p *machinel return "", err } - // Return the appropriate set of config based on whether http and/or grpc is active - if engine_http_port > 0 && engine_grpc_port > 0 { - if utils.GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" { - return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal + YAML_SEP + cTLSGlobal + YAML_SEP + cRestNamespaced + YAML_SEP + cGrpcNamespaced + YAML_SEP + cTLSNamespaced, nil - } else { - return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal + YAML_SEP + cTLSGlobal, nil - } - } else if engine_http_port > 0 { - if utils.GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" { - return YAML_SEP + cRestGlobal + YAML_SEP + cTLSGlobal + YAML_SEP + cRestNamespaced + YAML_SEP + cTLSNamespaced, nil - } else { - return YAML_SEP + YAML_SEP + cRestGlobal + YAML_SEP + cTLSGlobal, nil - } - } else if engine_grpc_port > 0 { - if utils.GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" { - return YAML_SEP + YAML_SEP + cGrpcGlobal + cTLSGlobal + YAML_SEP + cGrpcNamespaced + YAML_SEP + cTLSNamespaced, nil - } else { - return YAML_SEP + cGrpcGlobal + YAML_SEP + cTLSGlobal, nil - } + if utils.GetEnv("AMBASSADOR_SINGLE_NAMESPACE", "false") == "true" { + return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal + YAML_SEP + cTLSGlobal + YAML_SEP + cRestNamespaced + YAML_SEP + cGrpcNamespaced + YAML_SEP + cTLSNamespaced, nil } else { - return "", nil + return YAML_SEP + cRestGlobal + YAML_SEP + cGrpcGlobal + YAML_SEP + cTLSGlobal, nil } - } } diff --git a/operator/controllers/mlserver.go b/operator/controllers/mlserver.go index 535bd1593b..dc12af5105 100644 --- a/operator/controllers/mlserver.go +++ b/operator/controllers/mlserver.go @@ -68,15 +68,8 @@ func getMLServerContainer(pu *machinelearningv1.PredictiveUnit) (*v1.Container, return nil, err } - httpPort, err := getMLServerPort(pu, machinelearningv1.REST) - if err != nil { - return nil, err - } - - grpcPort, err := getMLServerPort(pu, machinelearningv1.GRPC) - if err != nil { - return nil, err - } + httpPort := pu.Endpoint.HttpPort + grpcPort := pu.Endpoint.GrpcPort cServer := &v1.Container{ Name: pu.Name, @@ -138,32 +131,23 @@ func getMLServerImage(pu *machinelearningv1.PredictiveUnit) (string, error) { return "", fmt.Errorf("failed to get server config for %s", *pu.Implementation) } - kfservingConfig := prepackConfig.Protocols.KFServing + if kfservingConfig, ok := prepackConfig.Protocols[machinelearningv1.ProtocolKfserving]; ok { + // Ignore version if empty + image := kfservingConfig.ContainerImage + if kfservingConfig.DefaultImageVersion != "" { + image = fmt.Sprintf("%s:%s", image, kfservingConfig.DefaultImageVersion) + } - if kfservingConfig == nil { + return image, nil + } else { err := fmt.Errorf("no image compatible with kfserving protocol for %s", *pu.Implementation) return "", err } - - // Ignore version if empty - image := kfservingConfig.ContainerImage - if kfservingConfig.DefaultImageVersion != "" { - image = fmt.Sprintf("%s:%s", image, kfservingConfig.DefaultImageVersion) - } - - return image, nil } func getMLServerEnvVars(pu *machinelearningv1.PredictiveUnit) ([]v1.EnvVar, error) { - httpPort, err := getMLServerPort(pu, machinelearningv1.REST) - if err != nil { - return nil, err - } - - grpcPort, err := getMLServerPort(pu, machinelearningv1.GRPC) - if err != nil { - return nil, err - } + httpPort := pu.Endpoint.HttpPort + grpcPort := pu.Endpoint.GrpcPort mlServerModelImplementation, err := getMLServerModelImplementation(pu) if err != nil { @@ -199,22 +183,6 @@ func getMLServerEnvVars(pu *machinelearningv1.PredictiveUnit) ([]v1.EnvVar, erro }, nil } -func getMLServerPort(pu *machinelearningv1.PredictiveUnit, endpointType machinelearningv1.EndpointType) (int32, error) { - if pu.Endpoint.Type == endpointType { - return pu.Endpoint.ServicePort, nil - } - - switch endpointType { - case machinelearningv1.REST: - return constants.MLServerDefaultHttpPort, nil - case machinelearningv1.GRPC: - return constants.MLServerDefaultGrpcPort, nil - } - - err := fmt.Errorf("invalid endpoint type: %s", endpointType) - return 0, err -} - func getMLServerModelImplementation(pu *machinelearningv1.PredictiveUnit) (string, error) { switch *pu.Implementation { case machinelearningv1.PrepackSklearnName: diff --git a/operator/controllers/mlserver_test.go b/operator/controllers/mlserver_test.go index f027a595ab..4fbfa89c11 100644 --- a/operator/controllers/mlserver_test.go +++ b/operator/controllers/mlserver_test.go @@ -3,7 +3,6 @@ package controllers import ( "fmt" machinelearningv1 "github.com/seldonio/seldon-core/operator/apis/machinelearning.seldon.io/v1" - "github.com/seldonio/seldon-core/operator/constants" v1 "k8s.io/api/core/v1" . "github.com/onsi/ginkgo" @@ -106,8 +105,8 @@ var _ = Describe("MLServer helpers", func() { } } - Expect(httpPort).To(Equal(fmt.Sprint(pu.Endpoint.ServicePort))) - Expect(grpcPort).To(Equal(fmt.Sprint(constants.MLServerDefaultGrpcPort))) + Expect(httpPort).To(Equal(fmt.Sprint(pu.Endpoint.HttpPort))) + Expect(grpcPort).To(Equal(fmt.Sprint(pu.Endpoint.GrpcPort))) }) It("adds the right model implementation and uri", func() { @@ -139,44 +138,6 @@ var _ = Describe("MLServer helpers", func() { }) }) - Describe("getMLServerPort", func() { - DescribeTable( - "returns the right port", - func(endpointType machinelearningv1.EndpointType, serviceEndpointType machinelearningv1.EndpointType, expected int32) { - pu.Endpoint.Type = serviceEndpointType - - port, err := getMLServerPort(pu, endpointType) - - Expect(err).NotTo(HaveOccurred()) - Expect(port).To(Equal(expected)) - }, - Entry( - "default httpPort", - machinelearningv1.REST, - machinelearningv1.GRPC, - constants.MLServerDefaultHttpPort, - ), - Entry( - "default grpcPort", - machinelearningv1.GRPC, - machinelearningv1.REST, - constants.MLServerDefaultGrpcPort, - ), - Entry( - "service httpPort", - machinelearningv1.REST, - machinelearningv1.REST, - int32(5001), - ), - Entry( - "service grpcPort", - machinelearningv1.GRPC, - machinelearningv1.GRPC, - int32(5001), - ), - ) - }) - Describe("getMLServerModelImplementation", func() { DescribeTable( "returns the right implementation", diff --git a/operator/controllers/seldondeployment_controller.go b/operator/controllers/seldondeployment_controller.go index d58d6f2d61..984f366cc5 100644 --- a/operator/controllers/seldondeployment_controller.go +++ b/operator/controllers/seldondeployment_controller.go @@ -205,9 +205,7 @@ func createPdb(podSpec *machinelearningv1.SeldonPodSpec, deploymentName string, func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, seldonId string, namespace string, - ports []httpGrpcPorts, - httpAllowed bool, - grpcAllowed bool) ([]*istio.VirtualService, []*istio.DestinationRule, error) { + ports []httpGrpcPorts) ([]*istio.VirtualService, []*istio.DestinationRule, error) { istio_gateway := utils.GetEnv(ENV_ISTIO_GATEWAY, "seldon-gateway") istioTLSMode := utils.GetEnv(ENV_ISTIO_TLS_MODE, "") @@ -375,20 +373,11 @@ func createIstioResources(mlDep *machinelearningv1.SeldonDeployment, httpVsvc.Spec.Http[0].Route = routesHttp grpcVsvc.Spec.Http[0].Route = routesGrpc - if httpAllowed && grpcAllowed { - vscs := make([]*istio.VirtualService, 2) - vscs[0] = httpVsvc - vscs[1] = grpcVsvc - return vscs, drules, nil - } else if httpAllowed { - vscs := make([]*istio.VirtualService, 1) - vscs[0] = httpVsvc - return vscs, drules, nil - } else { - vscs := make([]*istio.VirtualService, 1) - vscs[0] = grpcVsvc - return vscs, drules, nil - } + vscs := make([]*istio.VirtualService, 2) + vscs[0] = httpVsvc + vscs[1] = grpcVsvc + return vscs, drules, nil + } func getEngineHttpPort() (engine_http_port int, err error) { @@ -438,24 +427,6 @@ func (r *SeldonDeploymentReconciler) createComponents(ctx context.Context, mlDep // If one of the predictors has noEngine then only one of http or grpc should be allowed dependent on // the type of the noEngine model: whether it is http or grpc externalPorts := make([]httpGrpcPorts, len(mlDep.Spec.Predictors)) - grpcAllowed := true - httpAllowed := true - // Attempt to set httpAllowed and grpcAllowed to false if we have an noEngine predictor - for i := 0; i < len(mlDep.Spec.Predictors); i++ { - p := mlDep.Spec.Predictors[i] - noEngine := strings.ToLower(p.Annotations[machinelearningv1.ANNOTATION_NO_ENGINE]) == "true" - if noEngine && len(p.ComponentSpecs) > 0 && len(p.ComponentSpecs[0].Spec.Containers) > 0 { - pu := machinelearningv1.GetPredictiveUnit(&p.Graph, p.ComponentSpecs[0].Spec.Containers[0].Name) - if pu != nil { - if pu.Endpoint != nil && pu.Endpoint.Type == machinelearningv1.GRPC { - httpAllowed = false - } - if pu.Endpoint == nil || pu.Endpoint.Type == machinelearningv1.REST { - grpcAllowed = false - } - } - } - } for i := 0; i < len(mlDep.Spec.Predictors); i++ { p := mlDep.Spec.Predictors[i] @@ -552,40 +523,23 @@ func (r *SeldonDeploymentReconciler) createComponents(ctx context.Context, mlDep deploy.Spec.Selector.MatchLabels[machinelearningv1.Label_seldon_app] = pSvcName deploy.Spec.Template.ObjectMeta.Labels[machinelearningv1.Label_seldon_app] = pSvcName - port := int(svc.Spec.Ports[0].Port) - - if svc.Spec.Ports[0].Name == "grpc" { - httpAllowed = false - externalPorts[i] = httpGrpcPorts{httpPort: 0, grpcPort: port} - psvc, err := createPredictorService(pSvcName, seldonId, &p, mlDep, 0, port, false, log) - if err != nil { - return nil, err - } - psvc = addLabelsToService(psvc, pu, &p) - - c.services = append(c.services, psvc) - - c.serviceDetails[pSvcName] = &machinelearningv1.ServiceStatus{ - SvcName: pSvcName, - GrpcEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(port), - } - } else { - externalPorts[i] = httpGrpcPorts{httpPort: port, grpcPort: 0} - grpcAllowed = false - psvc, err := createPredictorService(pSvcName, seldonId, &p, mlDep, port, 0, false, log) - if err != nil { - return nil, err - } - psvc = addLabelsToService(psvc, pu, &p) - - c.services = append(c.services, psvc) - - c.serviceDetails[pSvcName] = &machinelearningv1.ServiceStatus{ - SvcName: pSvcName, - HttpEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(port), - } + httpPort := int(svc.Spec.Ports[0].Port) + grpcPort := int(svc.Spec.Ports[1].Port) + + externalPorts[i] = httpGrpcPorts{httpPort: httpPort, grpcPort: grpcPort} + psvc, err := createPredictorService(pSvcName, seldonId, &p, mlDep, httpPort, grpcPort, false, log) + if err != nil { + return nil, err } + psvc = addLabelsToService(psvc, pu, &p) + + c.services = append(c.services, psvc) + c.serviceDetails[pSvcName] = &machinelearningv1.ServiceStatus{ + SvcName: pSvcName, + HttpEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(httpPort), + GrpcEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(grpcPort), + } } } } @@ -648,41 +602,20 @@ func (r *SeldonDeploymentReconciler) createComponents(ctx context.Context, mlDep } } - //Create Service for Predictor - exposed externally (ambassador or istio) and points at engine - httpPort := engine_http_port - if httpAllowed == false { - httpPort = 0 - } - grpcPort := engine_grpc_port - if grpcAllowed == false { - grpcPort = 0 - } - psvc, err := createPredictorService(pSvcName, seldonId, &p, mlDep, httpPort, grpcPort, false, log) + psvc, err := createPredictorService(pSvcName, seldonId, &p, mlDep, engine_http_port, engine_grpc_port, false, log) if err != nil { return nil, err } c.services = append(c.services, psvc) - if httpAllowed && grpcAllowed { - c.serviceDetails[pSvcName] = &machinelearningv1.ServiceStatus{ - SvcName: pSvcName, - HttpEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(engine_http_port), - GrpcEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(engine_grpc_port), - } - } else if httpAllowed { - c.serviceDetails[pSvcName] = &machinelearningv1.ServiceStatus{ - SvcName: pSvcName, - HttpEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(engine_http_port), - } - } else if grpcAllowed { - c.serviceDetails[pSvcName] = &machinelearningv1.ServiceStatus{ - SvcName: pSvcName, - GrpcEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(engine_grpc_port), - } + c.serviceDetails[pSvcName] = &machinelearningv1.ServiceStatus{ + SvcName: pSvcName, + HttpEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(engine_http_port), + GrpcEndpoint: pSvcName + "." + namespace + ":" + strconv.Itoa(engine_grpc_port), } - externalPorts[i] = httpGrpcPorts{httpPort: httpPort, grpcPort: grpcPort} + externalPorts[i] = httpGrpcPorts{httpPort: engine_http_port, grpcPort: engine_grpc_port} } ei := NewExplainerInitializer(ctx, r.ClientSet) @@ -700,7 +633,7 @@ func (r *SeldonDeploymentReconciler) createComponents(ctx context.Context, mlDep //TODO Fixme - not changed to handle per predictor scenario if utils.GetEnv(ENV_ISTIO_ENABLED, "false") == "true" { - vsvcs, dstRule, err := createIstioResources(mlDep, seldonId, namespace, externalPorts, httpAllowed, grpcAllowed) + vsvcs, dstRule, err := createIstioResources(mlDep, seldonId, namespace, externalPorts) if err != nil { return nil, err } @@ -775,38 +708,11 @@ func createContainerService(deploy *appsv1.Deployment, return nil } namespace := getNamespace(mlDep) - portType := "http" - if pu.Endpoint.Type == machinelearningv1.GRPC { - portType = "grpc" - } - var portNum int32 - portNum = 0 - existingPort := machinelearningv1.GetPort(portType, con.Ports) - if existingPort != nil { - portNum = existingPort.ContainerPort - } - // pu should have a port set by seldondeployment_create_update_handler.go (if not by user) - // that mutator modifies SeldonDeployment and fires before this controller - if pu.Endpoint.ServicePort != 0 { - portNum = pu.Endpoint.ServicePort - } - - if portNum == 0 { - // should have port by now - // if we don't know what it would respond to so can't create a service for it - return nil - } - - if portType == "grpc" { - c.serviceDetails[containerServiceValue] = &machinelearningv1.ServiceStatus{ - SvcName: containerServiceValue, - GrpcEndpoint: containerServiceValue + "." + namespace + ":" + strconv.Itoa(int(portNum))} - } else { - c.serviceDetails[containerServiceValue] = &machinelearningv1.ServiceStatus{ - SvcName: containerServiceValue, - HttpEndpoint: containerServiceValue + "." + namespace + ":" + strconv.Itoa(int(portNum))} - } + c.serviceDetails[containerServiceValue] = &machinelearningv1.ServiceStatus{ + SvcName: containerServiceValue, + HttpEndpoint: containerServiceValue + "." + namespace + ":" + strconv.Itoa(int(pu.Endpoint.HttpPort)), + GrpcEndpoint: containerServiceValue + "." + namespace + ":" + strconv.Itoa(int(pu.Endpoint.GrpcPort))} svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -818,9 +724,15 @@ func createContainerService(deploy *appsv1.Deployment, Ports: []corev1.ServicePort{ { Protocol: corev1.ProtocolTCP, - Port: portNum, - TargetPort: intstr.FromInt(int(portNum)), - Name: portType, + Port: pu.Endpoint.HttpPort, + TargetPort: intstr.FromInt(int(pu.Endpoint.HttpPort)), + Name: "http", + }, + { + Protocol: corev1.ProtocolTCP, + Port: pu.Endpoint.GrpcPort, + TargetPort: intstr.FromInt(int(pu.Endpoint.GrpcPort)), + Name: "grpc", }, }, Type: corev1.ServiceTypeClusterIP, @@ -832,15 +744,31 @@ func createContainerService(deploy *appsv1.Deployment, deploy.Spec.Selector.MatchLabels[containerServiceKey] = containerServiceValue deploy.Spec.Template.ObjectMeta.Labels[containerServiceKey] = containerServiceValue - if existingPort == nil || con.Ports == nil { - con.Ports = append(con.Ports, corev1.ContainerPort{Name: portType, ContainerPort: portNum, Protocol: corev1.ProtocolTCP}) + existingHttpPort := machinelearningv1.GetPort("http", con.Ports) + if existingHttpPort == nil || con.Ports == nil { + con.Ports = append(con.Ports, corev1.ContainerPort{Name: "http", ContainerPort: pu.Endpoint.HttpPort, Protocol: corev1.ProtocolTCP}) + } + existingGrpcPort := machinelearningv1.GetPort("grpc", con.Ports) + if existingGrpcPort == nil || con.Ports == nil { + con.Ports = append(con.Ports, corev1.ContainerPort{Name: "grpc", ContainerPort: pu.Endpoint.GrpcPort, Protocol: corev1.ProtocolTCP}) } + // Backwards compatible additions. From 1.5.0 onwards could always call httpPort as both should be available but for + // previously wrapped components need to look at transport. + // TODO: deprecate and just call httpPort if con.LivenessProbe == nil { - con.LivenessProbe = &corev1.Probe{Handler: corev1.Handler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString(portType)}}, InitialDelaySeconds: 60, PeriodSeconds: 5, SuccessThreshold: 1, FailureThreshold: 3, TimeoutSeconds: 1} + if mlDep.Spec.Transport == machinelearningv1.TransportGrpc || pu.Endpoint.Type == machinelearningv1.GRPC { + con.LivenessProbe = &corev1.Probe{Handler: corev1.Handler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(int(pu.Endpoint.GrpcPort))}}, InitialDelaySeconds: 60, PeriodSeconds: 5, SuccessThreshold: 1, FailureThreshold: 3, TimeoutSeconds: 1} + } else { + con.LivenessProbe = &corev1.Probe{Handler: corev1.Handler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(int(pu.Endpoint.HttpPort))}}, InitialDelaySeconds: 60, PeriodSeconds: 5, SuccessThreshold: 1, FailureThreshold: 3, TimeoutSeconds: 1} + } } if con.ReadinessProbe == nil { - con.ReadinessProbe = &corev1.Probe{Handler: corev1.Handler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromString(portType)}}, InitialDelaySeconds: 20, PeriodSeconds: 5, SuccessThreshold: 1, FailureThreshold: 3, TimeoutSeconds: 1} + if mlDep.Spec.Transport == machinelearningv1.TransportGrpc || pu.Endpoint.Type == machinelearningv1.GRPC { + con.ReadinessProbe = &corev1.Probe{Handler: corev1.Handler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(int(pu.Endpoint.GrpcPort))}}, InitialDelaySeconds: 20, PeriodSeconds: 5, SuccessThreshold: 1, FailureThreshold: 3, TimeoutSeconds: 1} + } else { + con.ReadinessProbe = &corev1.Probe{Handler: corev1.Handler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt(int(pu.Endpoint.HttpPort))}}, InitialDelaySeconds: 20, PeriodSeconds: 5, SuccessThreshold: 1, FailureThreshold: 3, TimeoutSeconds: 1} + } } // Add livecycle probe @@ -848,9 +776,23 @@ func createContainerService(deploy *appsv1.Deployment, con.Lifecycle = &corev1.Lifecycle{PreStop: &corev1.Handler{Exec: &corev1.ExecAction{Command: []string{"/bin/sh", "-c", "/bin/sleep 10"}}}} } - // Add Environment Variables + // + // Backwards compatibility - set to either Http or Grpc + // + // TODO: deprecate and remove if !utils.HasEnvVar(con.Env, machinelearningv1.ENV_PREDICTIVE_UNIT_SERVICE_PORT) { - con.Env = append(con.Env, corev1.EnvVar{Name: machinelearningv1.ENV_PREDICTIVE_UNIT_SERVICE_PORT, Value: strconv.Itoa(int(portNum))}) + if pu.Endpoint.Type == machinelearningv1.GRPC || mlDep.Spec.Transport == machinelearningv1.TransportGrpc { + con.Env = append(con.Env, corev1.EnvVar{Name: machinelearningv1.ENV_PREDICTIVE_UNIT_SERVICE_PORT, Value: strconv.Itoa(int(pu.Endpoint.GrpcPort))}) + } else { + con.Env = append(con.Env, corev1.EnvVar{Name: machinelearningv1.ENV_PREDICTIVE_UNIT_SERVICE_PORT, Value: strconv.Itoa(int(pu.Endpoint.HttpPort))}) + } + } + + if !utils.HasEnvVar(con.Env, machinelearningv1.ENV_PREDICTIVE_UNIT_HTTP_SERVICE_PORT) { + con.Env = append(con.Env, corev1.EnvVar{Name: machinelearningv1.ENV_PREDICTIVE_UNIT_HTTP_SERVICE_PORT, Value: strconv.Itoa(int(pu.Endpoint.HttpPort))}) + } + if !utils.HasEnvVar(con.Env, machinelearningv1.ENV_PREDICTIVE_UNIT_GRPC_SERVICE_PORT) { + con.Env = append(con.Env, corev1.EnvVar{Name: machinelearningv1.ENV_PREDICTIVE_UNIT_GRPC_SERVICE_PORT, Value: strconv.Itoa(int(pu.Endpoint.GrpcPort))}) } if pu != nil && len(pu.Parameters) > 0 { diff --git a/operator/controllers/seldondeployment_engine.go b/operator/controllers/seldondeployment_engine.go index fb41853735..f6706241eb 100644 --- a/operator/controllers/seldondeployment_engine.go +++ b/operator/controllers/seldondeployment_engine.go @@ -225,16 +225,6 @@ func getSvcOrchUser(mlDep *machinelearningv1.SeldonDeployment) (*int64, error) { } func createExecutorContainer(mlDep *machinelearningv1.SeldonDeployment, p *machinelearningv1.PredictorSpec, predictorB64 string, http_port int, grpc_port int, resources *corev1.ResourceRequirements) (*corev1.Container, error) { - transport := mlDep.Spec.Transport - //Backwards compatible with older resources - if transport == "" { - if p.Graph.Endpoint.Type == machinelearningv1.GRPC { - transport = machinelearningv1.TransportGrpc - } else { - transport = machinelearningv1.TransportRest - } - } - protocol := mlDep.Spec.Protocol //Backwards compatibility for older resources if protocol == "" { @@ -268,7 +258,6 @@ func createExecutorContainer(mlDep *machinelearningv1.SeldonDeployment, p *machi "--predictor", p.Name, "--http_port", strconv.Itoa(http_port), "--grpc_port", strconv.Itoa(grpc_port), - "--transport", string(transport), "--protocol", string(protocol), "--prometheus_path", getPrometheusPath(mlDep), "--server_type", string(serverType), diff --git a/operator/controllers/seldondeployment_prepackaged_servers.go b/operator/controllers/seldondeployment_prepackaged_servers.go index fb1fd39581..c71029dac6 100644 --- a/operator/controllers/seldondeployment_prepackaged_servers.go +++ b/operator/controllers/seldondeployment_prepackaged_servers.go @@ -59,23 +59,17 @@ func extractEnvSecretRefName(pu *machinelearningv1.PredictiveUnit) string { return envSecretRefName } -func createTensorflowServingContainer(pu *machinelearningv1.PredictiveUnit, tensorflowProtocol bool) *v1.Container { +func createTensorflowServingContainer(mlDepSepc *machinelearningv1.SeldonDeploymentSpec, pu *machinelearningv1.PredictiveUnit, tensorflowProtocol bool) *v1.Container { ServerConfig := machinelearningv1.GetPrepackServerConfig(string(*pu.Implementation)) - tfImage := "tensorflow/serving:latest" - if ServerConfig.TensorflowImage != "" { - tfImage = ServerConfig.TensorflowImage - } + tfImage := ServerConfig.PrepackImageName(machinelearningv1.ProtocolTensorflow, pu) grpcPort := int32(constants.TfServingGrpcPort) restPort := int32(constants.TfServingRestPort) name := constants.TFServingContainerName if tensorflowProtocol { - if pu.Endpoint.Type == machinelearningv1.GRPC { - grpcPort = pu.Endpoint.ServicePort - } else { - restPort = pu.Endpoint.ServicePort - } + grpcPort = pu.Endpoint.GrpcPort + restPort = pu.Endpoint.HttpPort name = pu.Name } @@ -101,28 +95,28 @@ func createTensorflowServingContainer(pu *machinelearningv1.PredictiveUnit, tens } } -func (pi *PrePackedInitialiser) addTFServerContainer(mlDep *machinelearningv1.SeldonDeployment, pu *machinelearningv1.PredictiveUnit, deploy *appsv1.Deployment, serverConfig *machinelearningv1.PredictorServerConfig) error { +func (pi *PrePackedInitialiser) addTFServerContainer(mlDepSpec *machinelearningv1.SeldonDeploymentSpec, pu *machinelearningv1.PredictiveUnit, deploy *appsv1.Deployment, serverConfig *machinelearningv1.PredictorServerConfig) error { ty := machinelearningv1.MODEL pu.Type = &ty c := utils.GetContainerForDeployment(deploy, pu.Name) var tfServingContainer *v1.Container - if mlDep.Spec.Protocol == machinelearningv1.ProtocolTensorflow { + if mlDepSpec.Protocol == machinelearningv1.ProtocolTensorflow { tfServingContainer = c } else { - machinelearningv1.SetImageNameForPrepackContainer(pu, c, serverConfig) + c.Image = serverConfig.PrepackImageName(mlDepSpec.Protocol, pu) SetUriParamsForTFServingProxyContainer(pu, c) tfServingContainer = utils.GetContainerForDeployment(deploy, constants.TFServingContainerName) } existing := tfServingContainer != nil if !existing { - tfServingContainer = createTensorflowServingContainer(pu, mlDep.Spec.Protocol == machinelearningv1.ProtocolTensorflow) + tfServingContainer = createTensorflowServingContainer(mlDepSpec, pu, mlDepSpec.Protocol == machinelearningv1.ProtocolTensorflow) deploy.Spec.Template.Spec.Containers = append(deploy.Spec.Template.Spec.Containers, *tfServingContainer) } else { // Update any missing fields - protoType := createTensorflowServingContainer(pu, mlDep.Spec.Protocol == machinelearningv1.ProtocolTensorflow) + protoType := createTensorflowServingContainer(mlDepSpec, pu, mlDepSpec.Protocol == machinelearningv1.ProtocolTensorflow) if tfServingContainer.Image == "" { tfServingContainer.Image = protoType.Image } @@ -144,37 +138,30 @@ func (pi *PrePackedInitialiser) addTFServerContainer(mlDep *machinelearningv1.Se return nil } -func (pi *PrePackedInitialiser) addTritonServer(pu *machinelearningv1.PredictiveUnit, deploy *appsv1.Deployment, serverConfig *machinelearningv1.PredictorServerConfig) error { +func (pi *PrePackedInitialiser) addTritonServer(mlDepSpec *machinelearningv1.SeldonDeploymentSpec, pu *machinelearningv1.PredictiveUnit, deploy *appsv1.Deployment, serverConfig *machinelearningv1.PredictorServerConfig) error { c := utils.GetContainerForDeployment(deploy, pu.Name) existing := c != nil - httpPort := int32(constants.TritonDefaultHttpPort) - grpcPort := int32(constants.TritonDefaultGrpcPort) - if pu.Endpoint.Type == machinelearningv1.GRPC { - grpcPort = pu.Endpoint.ServicePort - } else { - httpPort = pu.Endpoint.ServicePort - } tritonUser := int64(1000) cServer := &v1.Container{ Name: pu.Name, Args: []string{ "/opt/tritonserver/bin/tritonserver", - "--grpc-port=" + strconv.Itoa(int(grpcPort)), - "--http-port=" + strconv.Itoa(int(httpPort)), + "--grpc-port=" + strconv.Itoa(int(pu.Endpoint.GrpcPort)), + "--http-port=" + strconv.Itoa(int(pu.Endpoint.HttpPort)), "--model-repository=" + DefaultModelLocalMountPath, }, Ports: []v1.ContainerPort{ { Name: "grpc", - ContainerPort: grpcPort, + ContainerPort: pu.Endpoint.GrpcPort, Protocol: v1.ProtocolTCP, }, { Name: "http", - ContainerPort: httpPort, + ContainerPort: pu.Endpoint.HttpPort, Protocol: v1.ProtocolTCP, }, }, @@ -210,7 +197,7 @@ func (pi *PrePackedInitialiser) addTritonServer(pu *machinelearningv1.Predictive }, }, } - machinelearningv1.SetImageNameForPrepackContainer(pu, cServer, serverConfig) + cServer.Image = serverConfig.PrepackImageName(mlDepSpec.Protocol, pu) if existing { // Overwrite core items if not existing or required @@ -278,7 +265,7 @@ func (pi *PrePackedInitialiser) addMLServerDefault(pu *machinelearningv1.Predict return nil } -func (pi *PrePackedInitialiser) addModelDefaultServers(pu *machinelearningv1.PredictiveUnit, deploy *appsv1.Deployment, serverConfig *machinelearningv1.PredictorServerConfig) error { +func (pi *PrePackedInitialiser) addModelDefaultServers(mlDepSepc *machinelearningv1.SeldonDeploymentSpec, pu *machinelearningv1.PredictiveUnit, deploy *appsv1.Deployment, serverConfig *machinelearningv1.PredictorServerConfig) error { ty := machinelearningv1.MODEL pu.Type = &ty @@ -299,7 +286,9 @@ func (pi *PrePackedInitialiser) addModelDefaultServers(pu *machinelearningv1.Pre } } - machinelearningv1.SetImageNameForPrepackContainer(pu, c, serverConfig) + if c.Image == "" { + c.Image = serverConfig.PrepackImageName(mlDepSepc.Protocol, pu) + } // Add parameters envvar - point at mount path because initContainer will download params := pu.Parameters @@ -350,41 +339,33 @@ func SetUriParamsForTFServingProxyContainer(pu *machinelearningv1.PredictiveUnit if len(pu.Parameters) > 0 { for _, paramElement := range pu.Parameters { if paramElement.Name == "rest_endpoint" || paramElement.Name == "grpc_endpoint" { - hasUriParams = true } } } if !hasUriParams { - var uriParam machinelearningv1.Parameter - - if pu.Endpoint.Type == machinelearningv1.REST { - uriParam = machinelearningv1.Parameter{ - Name: "rest_endpoint", - Type: "STRING", - Value: "http://0.0.0.0:2001", - } - } else { - uriParam = machinelearningv1.Parameter{ - Name: "grpc_endpoint", - Type: "STRING", - Value: "0.0.0.0:2000", - } - + uriParam := machinelearningv1.Parameter{ + Name: "rest_endpoint", + Type: "STRING", + Value: "http://0.0.0.0:2001", } - - parameters = append(pu.Parameters, uriParam) - - modelNameParam := machinelearningv1.Parameter{ - Name: "model_name", + parameters = append(parameters, uriParam) + uriParam = machinelearningv1.Parameter{ + Name: "grpc_endpoint", Type: "STRING", - Value: pu.Name, + Value: "0.0.0.0:2000", } + parameters = append(parameters, uriParam) + } - parameters = append(parameters, modelNameParam) - + modelNameParam := machinelearningv1.Parameter{ + Name: "model_name", + Type: "STRING", + Value: pu.Name, } + parameters = append(parameters, modelNameParam) + if len(parameters) > 0 { if !utils.HasEnvVar(c.Env, machinelearningv1.ENV_PREDICTIVE_UNIT_PARAMETERS) { c.Env = append(c.Env, v1.EnvVar{Name: machinelearningv1.ENV_PREDICTIVE_UNIT_PARAMETERS, Value: utils.GetPredictiveUnitAsJson(parameters)}) @@ -431,11 +412,11 @@ func (pi *PrePackedInitialiser) createStandaloneModelServers(mlDep *machinelearn if serverConfig != nil { switch *pu.Implementation { case machinelearningv1.PrepackTensorflowName: - if err := pi.addTFServerContainer(mlDep, pu, deploy, serverConfig); err != nil { + if err := pi.addTFServerContainer(&mlDep.Spec, pu, deploy, serverConfig); err != nil { return err } case machinelearningv1.PrepackTritonName: - if err := pi.addTritonServer(pu, deploy, serverConfig); err != nil { + if err := pi.addTritonServer(&mlDep.Spec, pu, deploy, serverConfig); err != nil { return err } default: @@ -446,7 +427,7 @@ func (pi *PrePackedInitialiser) createStandaloneModelServers(mlDep *machinelearn return err } } else { - if err := pi.addModelDefaultServers(pu, deploy, serverConfig); err != nil { + if err := pi.addModelDefaultServers(&mlDep.Spec, pu, deploy, serverConfig); err != nil { return err } } diff --git a/operator/controllers/seldondeployment_prepackaged_servers_test.go b/operator/controllers/seldondeployment_prepackaged_servers_test.go index af47f4826d..db32c7af56 100644 --- a/operator/controllers/seldondeployment_prepackaged_servers_test.go +++ b/operator/controllers/seldondeployment_prepackaged_servers_test.go @@ -252,10 +252,10 @@ var _ = Describe("Create a prepacked tfserving server for tensorflow protocol an if c.Name == modelName { for _, arg := range c.Args { if strings.Index(arg, constants.TfServingArgPort) == 0 { - Expect(arg).To(Equal(constants.TfServingArgPort + strconv.Itoa(constants.TfServingGrpcPort))) + Expect(arg).To(Equal(constants.TfServingArgPort + strconv.Itoa(int(constants.FirstGrpcPortNumber)))) } if strings.Index(arg, constants.TfServingArgRestPort) == 0 { - Expect(arg).To(Equal(constants.TfServingArgRestPort + strconv.Itoa(int(constants.FirstPortNumber)))) + Expect(arg).To(Equal(constants.TfServingArgRestPort + strconv.Itoa(int(constants.FirstHttpPortNumber)))) } } } @@ -358,10 +358,10 @@ var _ = Describe("Create a prepacked tfserving server for tensorflow protocol an Expect(*c.Resources.Requests.Cpu()).To(Equal(cpuRequest)) for _, arg := range c.Args { if strings.Index(arg, constants.TfServingArgPort) == 0 { - Expect(arg).To(Equal(constants.TfServingArgPort + strconv.Itoa(constants.TfServingGrpcPort))) + Expect(arg).To(Equal(constants.TfServingArgPort + strconv.Itoa(int(constants.FirstGrpcPortNumber)))) } if strings.Index(arg, constants.TfServingArgRestPort) == 0 { - Expect(arg).To(Equal(constants.TfServingArgRestPort + strconv.Itoa(int(constants.FirstPortNumber)))) + Expect(arg).To(Equal(constants.TfServingArgRestPort + strconv.Itoa(int(constants.FirstHttpPortNumber)))) } } } @@ -485,10 +485,10 @@ var _ = Describe("Create a prepacked tfserving server for tensorflow protocol an if c.Name == modelName { for _, arg := range c.Args { if strings.Index(arg, constants.TfServingArgPort) == 0 { - Expect(arg).To(Equal(constants.TfServingArgPort + strconv.Itoa(int(constants.FirstPortNumber)))) + Expect(arg).To(Equal(constants.TfServingArgPort + strconv.Itoa(int(constants.FirstGrpcPortNumber)))) } if strings.Index(arg, constants.TfServingArgRestPort) == 0 { - Expect(arg).To(Equal(constants.TfServingArgRestPort + strconv.Itoa(constants.TfServingRestPort))) + Expect(arg).To(Equal(constants.TfServingArgRestPort + strconv.Itoa(int(constants.FirstHttpPortNumber)))) } } } @@ -647,7 +647,8 @@ var _ = Describe("Create a prepacked triton server", func() { Namespace: key.Namespace, }, Spec: machinelearningv1.SeldonDeploymentSpec{ - Name: name, + Name: name, + Protocol: machinelearningv1.ProtocolKfserving, Predictors: []machinelearningv1.PredictorSpec{ { Name: "p1", @@ -720,3 +721,89 @@ var _ = Describe("Create a prepacked triton server", func() { }) }) + +var _ = Describe("Create a prepacked mlflow server with existing container", func() { + const timeout = time.Second * 30 + const interval = time.Second * 1 + const name = "pp1" + const sdepName = "prepack1" + envExecutorUser = "2" + By("Creating a resource") + It("should create a resource with defaults", func() { + Expect(k8sClient).NotTo(BeNil()) + var modelType = machinelearningv1.MODEL + var impl = machinelearningv1.PredictiveUnitImplementation(constants.PrePackedMlflow) + key := types.NamespacedName{ + Name: sdepName, + Namespace: "default", + } + instance := &machinelearningv1.SeldonDeployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + }, + Spec: machinelearningv1.SeldonDeploymentSpec{ + Name: name, + Predictors: []machinelearningv1.PredictorSpec{ + { + Name: "p1", + ComponentSpecs: []*machinelearningv1.SeldonPodSpec{ + { + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "classifier", + }, + }, + }, + }, + }, + Graph: machinelearningv1.PredictiveUnit{ + Name: "classifier", + Type: &modelType, + Implementation: &impl, + Endpoint: &machinelearningv1.Endpoint{Type: machinelearningv1.REST}, + }, + }, + }, + }, + } + + configMapName := types.NamespacedName{Name: "seldon-config", + Namespace: "seldon-system"} + + configResult := &corev1.ConfigMap{} + const timeout = time.Second * 30 + Eventually(func() error { return k8sClient.Get(context.TODO(), configMapName, configResult) }, timeout). + Should(Succeed()) + + // Run Defaulter + instance.Default() + + Expect(k8sClient.Create(context.Background(), instance)).Should(Succeed()) + //time.Sleep(time.Second * 5) + + fetched := &machinelearningv1.SeldonDeployment{} + Eventually(func() error { + err := k8sClient.Get(context.Background(), key, fetched) + return err + }, timeout, interval).Should(BeNil()) + Expect(fetched.Name).Should(Equal(sdepName)) + + sPodSpec, idx := utils.GetSeldonPodSpecForPredictiveUnit(&instance.Spec.Predictors[0], instance.Spec.Predictors[0].Graph.Name) + depName := machinelearningv1.GetDeploymentName(instance, instance.Spec.Predictors[0], sPodSpec, idx) + depKey := types.NamespacedName{ + Name: depName, + Namespace: "default", + } + depFetched := &appsv1.Deployment{} + Eventually(func() error { + err := k8sClient.Get(context.Background(), depKey, depFetched) + return err + }, timeout, interval).Should(BeNil()) + Expect(len(depFetched.Spec.Template.Spec.Containers)).Should(Equal(2)) + + Expect(k8sClient.Delete(context.Background(), instance)).Should(Succeed()) + }) + +}) diff --git a/operator/controllers/suite_test.go b/operator/controllers/suite_test.go index 67cac6641c..a0ccc3475c 100644 --- a/operator/controllers/suite_test.go +++ b/operator/controllers/suite_test.go @@ -65,65 +65,63 @@ func TestAPIs(t *testing.T) { var configs = map[string]string{ "predictor_servers": `{ - "TENSORFLOW_SERVER": { - "tensorflow": true, - "tfImage": "tensorflow/serving:2.1", - "rest": { - "image": "seldonio/tfserving-proxy_rest", - "defaultImageVersion": "0.7" - }, - "grpc": { - "image": "seldonio/tfserving-proxy_grpc", - "defaultImageVersion": "0.7" - } - }, - "SKLEARN_SERVER": { - "rest": { - "image": "seldonio/sklearnserver_rest", - "defaultImageVersion": "0.2" - }, - "grpc": { - "image": "seldonio/sklearnserver_grpc", - "defaultImageVersion": "0.2" - }, - "protocols": { - "kfserving": { - "image": "seldonio/mlserver", - "defaultImageVersion": "0.1.0" - } - } - }, - "XGBOOST_SERVER": { - "rest": { - "image": "seldonio/xgboostserver_rest", - "defaultImageVersion": "0.2" - }, - "grpc": { - "image": "seldonio/xgboostserver_grpc", - "defaultImageVersion": "0.2" - } - }, - "MLFLOW_SERVER": { - "rest": { - "image": "seldonio/mlflowserver_rest", - "defaultImageVersion": "0.2" - }, - "grpc": { - "image": "seldonio/mlflowserver_grpc", - "defaultImageVersion": "0.2" - } - }, - "TRITON_SERVER": { - "rest": { - "image": "nvcr.io/nvidia/tritonserver", - "defaultImageVersion": "20.08-py3" - }, - "grpc": { - "image": "nvcr.io/nvidia/tritonserver", - "defaultImageVersion": "20.08-py3" - } - } - }`, + "TENSORFLOW_SERVER": { + "protocols" : { + "tensorflow": { + "image": "tensorflow/serving", + "defaultImageVersion": "2.1.0" + }, + "seldon": { + "image": "seldonio/tfserving-proxy", + "defaultImageVersion": "1.3.0-dev" + } + } + }, + "SKLEARN_SERVER": { + "protocols" : { + "seldon": { + "image": "seldonio/sklearnserver", + "defaultImageVersion": "1.3.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } + } + }, + "XGBOOST_SERVER": { + "protocols" : { + "seldon": { + "image": "seldonio/xgboostserver", + "defaultImageVersion": "1.3.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } + } + }, + "MLFLOW_SERVER": { + "protocols" : { + "seldon": { + "image": "seldonio/mlflowserver", + "defaultImageVersion": "1.3.0-dev" + }, + "kfserving": { + "image": "seldonio/mlserver", + "defaultImageVersion": "0.1.0" + } + } + }, + "TRITON_SERVER": { + "protocols" : { + "kfserving": { + "image": "nvcr.io/nvidia/tritonserver", + "defaultImageVersion": "20.08-py3" + } + } + } + }`, "storageInitializer": ` { "image" : "gcr.io/kfserving/storage-initializer:v0.4.0", diff --git a/operator/testing/machinelearning.seldon.io_seldondeployments.yaml b/operator/testing/machinelearning.seldon.io_seldondeployments.yaml index d53a8a8974..112f90e8e6 100644 --- a/operator/testing/machinelearning.seldon.io_seldondeployments.yaml +++ b/operator/testing/machinelearning.seldon.io_seldondeployments.yaml @@ -4,6 +4,7 @@ metadata: annotations: cert-manager.io/inject-ca-from: seldon-system/seldon-serving-cert controller-gen.kubebuilder.io/version: v0.2.5 + creationTimestamp: null labels: app: seldon app.kubernetes.io/instance: seldon1 @@ -30,10 +31,14 @@ spec: description: SeldonDeployment is the Schema for the seldondeployments API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string metadata: type: object @@ -68,30 +73,56 @@ spec: type: integer metrics: items: - description: MetricSpec specifies how to scale based on a single metric (only `type` and one other matching field should be set at once). + description: MetricSpec specifies how to scale based + on a single metric (only `type` and one other matching + field should be set at once). properties: external: - description: external refers to a global metric that is not associated with any Kubernetes object. It allows autoscaling based on information coming from components running outside of cluster (for example length of queue in cloud messaging service, or QPS from loadbalancer running outside of cluster). + description: external refers to a global metric + that is not associated with any Kubernetes object. + It allows autoscaling based on information coming + from components running outside of cluster (for + example length of queue in cloud messaging service, + or QPS from loadbalancer running outside of + cluster). properties: metricName: - description: metricName is the name of the metric in question. + description: metricName is the name of the + metric in question. type: string metricSelector: - description: metricSelector is used to identify a specific time series within a given metric. + description: metricSelector is used to identify + a specific time series within a given metric. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -103,55 +134,91 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object targetAverageValue: anyOf: - type: integer - type: string - description: targetAverageValue is the target per-pod value of global metric (as a quantity). Mutually exclusive with TargetValue. + description: targetAverageValue is the target + per-pod value of global metric (as a quantity). + Mutually exclusive with TargetValue. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true targetValue: anyOf: - type: integer - type: string - description: targetValue is the target value of the metric (as a quantity). Mutually exclusive with TargetAverageValue. + description: targetValue is the target value + of the metric (as a quantity). Mutually + exclusive with TargetAverageValue. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: - metricName type: object object: - description: object refers to a metric describing a single kubernetes object (for example, hits-per-second on an Ingress object). + description: object refers to a metric describing + a single kubernetes object (for example, hits-per-second + on an Ingress object). properties: averageValue: anyOf: - type: integer - type: string - description: averageValue is the target value of the average of the metric across all relevant pods (as a quantity) + description: averageValue is the target value + of the average of the metric across all + relevant pods (as a quantity) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true metricName: - description: metricName is the name of the metric in question. + description: metricName is the name of the + metric in question. type: string selector: - description: selector is the string-encoded form of a standard kubernetes label selector for the given metric When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping When unset, just the metricName will be used to gather metrics. + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping + When unset, just the metricName will be + used to gather metrics. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -163,20 +230,28 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object target: - description: target is the described Kubernetes object. + description: target is the described Kubernetes + object. properties: apiVersion: description: API version of the referent type: string kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + description: 'Kind of the referent; More + info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + description: 'Name of the referent; More + info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string required: - kind @@ -186,7 +261,8 @@ spec: anyOf: - type: integer - type: string - description: targetValue is the target value of the metric (as a quantity). + description: targetValue is the target value + of the metric (as a quantity). pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: @@ -195,27 +271,54 @@ spec: - targetValue type: object pods: - description: pods refers to a metric describing each pod in the current scale target (for example, transactions-processed-per-second). The values will be averaged together before being compared to the target value. + description: pods refers to a metric describing + each pod in the current scale target (for example, + transactions-processed-per-second). The values + will be averaged together before being compared + to the target value. properties: metricName: - description: metricName is the name of the metric in question + description: metricName is the name of the + metric in question type: string selector: - description: selector is the string-encoded form of a standard kubernetes label selector for the given metric When set, it is passed as an additional parameter to the metrics server for more specific metrics scoping When unset, just the metricName will be used to gather metrics. + description: selector is the string-encoded + form of a standard kubernetes label selector + for the given metric When set, it is passed + as an additional parameter to the metrics + server for more specific metrics scoping + When unset, just the metricName will be + used to gather metrics. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key + that the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -227,14 +330,21 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. type: object type: object targetAverageValue: anyOf: - type: integer - type: string - description: targetAverageValue is the target value of the average of the metric across all relevant pods (as a quantity) + description: targetAverageValue is the target + value of the average of the metric across + all relevant pods (as a quantity) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: @@ -242,27 +352,44 @@ spec: - targetAverageValue type: object resource: - description: resource refers to a resource metric (such as those specified in requests and limits) known to Kubernetes describing each pod in the current scale target (e.g. CPU or memory). Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the "pods" source. + description: resource refers to a resource metric + (such as those specified in requests and limits) + known to Kubernetes describing each pod in the + current scale target (e.g. CPU or memory). Such + metrics are built in to Kubernetes, and have + special scaling options on top of those available + to normal per-pod metrics using the "pods" source. properties: name: - description: name is the name of the resource in question. + description: name is the name of the resource + in question. type: string targetAverageUtilization: - description: targetAverageUtilization is the target value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. + description: targetAverageUtilization is the + target value of the average of the resource + metric across all relevant pods, represented + as a percentage of the requested value of + the resource for the pods. format: int32 type: integer targetAverageValue: anyOf: - type: integer - type: string - description: targetAverageValue is the target value of the average of the resource metric across all relevant pods, as a raw value (instead of as a percentage of the request), similar to the "pods" metric source type. + description: targetAverageValue is the target + value of the average of the resource metric + across all relevant pods, as a raw value + (instead of as a percentage of the request), + similar to the "pods" metric source type. pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: - name type: object type: - description: type is the type of metric source. It should be one of "Object", "Pods" or "Resource", each mapping to a matching field in the object. + description: type is the type of metric source. It + should be one of "Object", "Pods" or "Resource", + each mapping to a matching field in the object. type: string required: - type @@ -275,34 +402,60 @@ spec: - maxReplicas type: object kedaSpec: - description: SeldonScaledObjectSpec is the spec for a KEDA ScaledObject resource + description: SeldonScaledObjectSpec is the spec for a KEDA + ScaledObject resource properties: advanced: - description: AdvancedConfig specifies advance scaling options + description: AdvancedConfig specifies advance scaling + options properties: horizontalPodAutoscalerConfig: - description: HorizontalPodAutoscalerConfig specifies horizontal scale config + description: HorizontalPodAutoscalerConfig specifies + horizontal scale config properties: behavior: - description: HorizontalPodAutoscalerBehavior configures the scaling behavior of the target in both Up and Down directions (scaleUp and scaleDown fields respectively). + description: HorizontalPodAutoscalerBehavior + configures the scaling behavior of the target + in both Up and Down directions (scaleUp and + scaleDown fields respectively). properties: scaleDown: - description: scaleDown is scaling policy for scaling Down. If not set, the default value is to allow to scale down to minReplicas pods, with a 300 second stabilization window (i.e., the highest recommendation for the last 300sec is used). + description: scaleDown is scaling policy + for scaling Down. If not set, the default + value is to allow to scale down to minReplicas + pods, with a 300 second stabilization + window (i.e., the highest recommendation + for the last 300sec is used). properties: policies: - description: policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + description: policies is a list of potential + scaling polices which can be used + during scaling. At least one policy + must be specified, otherwise the HPAScalingRules + will be discarded as invalid items: - description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + description: HPAScalingPolicy is a + single policy which must hold true + for a specified past interval. properties: periodSeconds: - description: PeriodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + description: PeriodSeconds specifies + the window of time for which + the policy should hold true. + PeriodSeconds must be greater + than zero and less than or equal + to 1800 (30 min). format: int32 type: integer type: - description: Type is used to specify the scaling policy. + description: Type is used to specify + the scaling policy. type: string value: - description: Value contains the amount of change which is permitted by the policy. It must be greater than zero + description: Value contains the + amount of change which is permitted + by the policy. It must be greater + than zero format: int32 type: integer required: @@ -312,30 +465,64 @@ spec: type: object type: array selectPolicy: - description: selectPolicy is used to specify which policy should be used. If not set, the default value MaxPolicySelect is used. + description: selectPolicy is used to + specify which policy should be used. + If not set, the default value MaxPolicySelect + is used. type: string stabilizationWindowSeconds: - description: 'StabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).' + description: 'StabilizationWindowSeconds + is the number of seconds for which + past recommendations should be considered + while scaling up or scaling down. + StabilizationWindowSeconds must be + greater than or equal to zero and + less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. + the stabilization window is 300 seconds + long).' format: int32 type: integer type: object scaleUp: - description: 'scaleUp is scaling policy for scaling Up. If not set, the default value is the higher of: * increase no more than 4 pods per 60 seconds * double the number of pods per 60 seconds No stabilization is used.' + description: 'scaleUp is scaling policy + for scaling Up. If not set, the default + value is the higher of: * increase no + more than 4 pods per 60 seconds * double + the number of pods per 60 seconds No stabilization + is used.' properties: policies: - description: policies is a list of potential scaling polices which can be used during scaling. At least one policy must be specified, otherwise the HPAScalingRules will be discarded as invalid + description: policies is a list of potential + scaling polices which can be used + during scaling. At least one policy + must be specified, otherwise the HPAScalingRules + will be discarded as invalid items: - description: HPAScalingPolicy is a single policy which must hold true for a specified past interval. + description: HPAScalingPolicy is a + single policy which must hold true + for a specified past interval. properties: periodSeconds: - description: PeriodSeconds specifies the window of time for which the policy should hold true. PeriodSeconds must be greater than zero and less than or equal to 1800 (30 min). + description: PeriodSeconds specifies + the window of time for which + the policy should hold true. + PeriodSeconds must be greater + than zero and less than or equal + to 1800 (30 min). format: int32 type: integer type: - description: Type is used to specify the scaling policy. + description: Type is used to specify + the scaling policy. type: string value: - description: Value contains the amount of change which is permitted by the policy. It must be greater than zero + description: Value contains the + amount of change which is permitted + by the policy. It must be greater + than zero format: int32 type: integer required: @@ -345,43 +532,82 @@ spec: type: object type: array selectPolicy: - description: selectPolicy is used to specify which policy should be used. If not set, the default value MaxPolicySelect is used. + description: selectPolicy is used to + specify which policy should be used. + If not set, the default value MaxPolicySelect + is used. type: string stabilizationWindowSeconds: - description: 'StabilizationWindowSeconds is the number of seconds for which past recommendations should be considered while scaling up or scaling down. StabilizationWindowSeconds must be greater than or equal to zero and less than or equal to 3600 (one hour). If not set, use the default values: - For scale up: 0 (i.e. no stabilization is done). - For scale down: 300 (i.e. the stabilization window is 300 seconds long).' + description: 'StabilizationWindowSeconds + is the number of seconds for which + past recommendations should be considered + while scaling up or scaling down. + StabilizationWindowSeconds must be + greater than or equal to zero and + less than or equal to 3600 (one hour). + If not set, use the default values: + - For scale up: 0 (i.e. no stabilization + is done). - For scale down: 300 (i.e. + the stabilization window is 300 seconds + long).' format: int32 type: integer type: object type: object resourceMetrics: items: - description: ResourceMetricSource indicates how to scale on a resource metric known to Kubernetes, as specified in requests and limits, describing each pod in the current scale target (e.g. CPU or memory). The values will be averaged together before being compared to the target. Such metrics are built in to Kubernetes, and have special scaling options on top of those available to normal per-pod metrics using the "pods" source. Only one "target" type should be set. + description: ResourceMetricSource indicates + how to scale on a resource metric known + to Kubernetes, as specified in requests + and limits, describing each pod in the current + scale target (e.g. CPU or memory). The + values will be averaged together before + being compared to the target. Such metrics + are built in to Kubernetes, and have special + scaling options on top of those available + to normal per-pod metrics using the "pods" + source. Only one "target" type should be + set. properties: name: - description: name is the name of the resource in question. + description: name is the name of the resource + in question. type: string target: - description: target specifies the target value for the given metric + description: target specifies the target + value for the given metric properties: averageUtilization: - description: averageUtilization is the target value of the average of the resource metric across all relevant pods, represented as a percentage of the requested value of the resource for the pods. Currently only valid for Resource metric source type + description: averageUtilization is + the target value of the average + of the resource metric across all + relevant pods, represented as a + percentage of the requested value + of the resource for the pods. Currently + only valid for Resource metric source + type format: int32 type: integer averageValue: anyOf: - type: integer - type: string - description: averageValue is the target value of the average of the metric across all relevant pods (as a quantity) + description: averageValue is the target + value of the average of the metric + across all relevant pods (as a quantity) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true type: - description: type represents whether the metric type is Utilization, Value, or AverageValue + description: type represents whether + the metric type is Utilization, + Value, or AverageValue type: string value: anyOf: - type: integer - type: string - description: value is the target value of the metric (as a quantity). + description: value is the target value + of the metric (as a quantity). pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true required: @@ -410,10 +636,13 @@ spec: type: integer triggers: items: - description: ScaleTriggers reference the scaler that will be used + description: ScaleTriggers reference the scaler that + will be used properties: authenticationRef: - description: ScaledObjectAuthRef points to the TriggerAuthentication object that is used to authenticate the scaler with the environment + description: ScaledObjectAuthRef points to the + TriggerAuthentication object that is used to + authenticate the scaler with the environment properties: name: type: string @@ -444,13 +673,23 @@ spec: anyOf: - type: integer - type: string - description: An eviction is allowed if at most "maxUnavailable" pods in the deployment corresponding to a componentSpec are unavailable after the eviction, i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions by specifying 0. MaxUnavailable and MinAvailable are mutually exclusive. + description: An eviction is allowed if at most "maxUnavailable" + pods in the deployment corresponding to a componentSpec + are unavailable after the eviction, i.e. even in absence + of the evicted pod. For example, one can prevent all + voluntary evictions by specifying 0. MaxUnavailable + and MinAvailable are mutually exclusive. x-kubernetes-int-or-string: true minAvailable: anyOf: - type: integer - type: string - description: An eviction is allowed if at least "minAvailable" pods in the deployment corresponding to a componentSpec will still be available after the eviction, i.e. even in the absence of the evicted pod. So for example you can prevent all voluntary evictions by specifying "100%". + description: An eviction is allowed if at least "minAvailable" + pods in the deployment corresponding to a componentSpec + will still be available after the eviction, i.e. even + in the absence of the evicted pod. So for example + you can prevent all voluntary evictions by specifying + "100%". x-kubernetes-int-or-string: true type: object replicas: @@ -460,36 +699,79 @@ spec: description: PodSpec is a description of a pod. properties: activeDeadlineSeconds: - description: Optional duration in seconds the pod may be active on the node relative to StartTime before the system will actively try to mark it failed and kill associated containers. Value must be a positive integer. + description: Optional duration in seconds the pod may + be active on the node relative to StartTime before + the system will actively try to mark it failed and + kill associated containers. Value must be a positive + integer. format: int64 type: integer affinity: description: If specified, the pod's scheduling constraints properties: nodeAffinity: - description: Describes node affinity scheduling rules for the pod. + description: Describes node affinity scheduling + rules for the pod. properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node matches the corresponding matchExpressions; the node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most + preferred. items: - description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + description: An empty preferred scheduling + term matches all objects with implicit weight + 0 (i.e. it's a no-op). A null preferred + scheduling term matches no objects (i.e. + is also a no-op). properties: preference: - description: A node selector term, associated with the corresponding weight. + description: A node selector term, associated + with the corresponding weight. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector + requirements by node's labels. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -499,18 +781,38 @@ spec: type: object type: array matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector + requirements by node's fields. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -521,7 +823,9 @@ spec: type: array type: object weight: - description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + description: Weight associated with matching + the corresponding nodeSelectorTerm, + in the range 1-100. format: int32 type: integer required: @@ -530,26 +834,57 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to an update), the + system may or may not try to eventually evict + the pod from its node. properties: nodeSelectorTerms: - description: Required. A list of node selector terms. The terms are ORed. + description: Required. A list of node selector + terms. The terms are ORed. items: - description: A null or empty node selector term matches no objects. The requirements of them are ANDed. The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + description: A null or empty node selector + term matches no objects. The requirements + of them are ANDed. The TopologySelectorTerm + type implements a subset of the NodeSelectorTerm. properties: matchExpressions: - description: A list of node selector requirements by node's labels. + description: A list of node selector + requirements by node's labels. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -559,18 +894,38 @@ spec: type: object type: array matchFields: - description: A list of node selector requirements by node's fields. + description: A list of node selector + requirements by node's fields. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A node selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: The label key that the selector applies to. + description: The label key that + the selector applies to. type: string operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: Represents a key's + relationship to a set of values. + Valid operators are In, NotIn, + Exists, DoesNotExist. Gt, + and Lt. type: string values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: An array of string + values. If the operator is + In or NotIn, the values array + must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. + If the operator is Gt or Lt, + the values array must have + a single element, which will + be interpreted as an integer. + This array is replaced during + a strategic merge patch. items: type: string type: array @@ -586,32 +941,74 @@ spec: type: object type: object podAffinity: - description: Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod affinity scheduling rules + (e.g. co-locate this pod in the same node, zone, + etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule + pods to nodes that satisfy the affinity expressions + specified by this field, but it may choose + a node that violates one or more of the expressions. + The node that is most preferred is the one + with the greatest sum of weights, i.e. for + each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum + by iterating through the elements of this + field and adding "weight" to the sum if the + node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest + sum are the most preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity + term, associated with the corresponding + weight. properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a + set of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the + label key that the selector + applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. items: type: string type: array @@ -623,22 +1020,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. format: int32 type: integer required: @@ -647,26 +1067,64 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: If the affinity requirements specified + by this field are not met at scheduling time, + the pod will not be scheduled onto the node. + If the affinity requirements specified by + this field cease to be met at some point during + pod execution (e.g. due to a pod label update), + the system may or may not try to eventually + evict the pod from its node. When there are + multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. + all terms must be satisfied. items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a set + of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label + key that the selector applies + to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. items: type: string type: array @@ -678,16 +1136,34 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey @@ -695,32 +1171,75 @@ spec: type: array type: object podAntiAffinity: - description: Describes pod anti-affinity scheduling rules (e.g. avoid putting this pod in the same node, zone, etc. as some other pod(s)). + description: Describes pod anti-affinity scheduling + rules (e.g. avoid putting this pod in the same + node, zone, etc. as some other pod(s)). properties: preferredDuringSchedulingIgnoredDuringExecution: - description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. The node that is most preferred is the one with the greatest sum of weights, i.e. for each node that meets all of the scheduling requirements (resource request, requiredDuringScheduling anti-affinity expressions, etc.), compute a sum by iterating through the elements of this field and adding "weight" to the sum if the node has pods which matches the corresponding podAffinityTerm; the node(s) with the highest sum are the most preferred. + description: The scheduler will prefer to schedule + pods to nodes that satisfy the anti-affinity + expressions specified by this field, but it + may choose a node that violates one or more + of the expressions. The node that is most + preferred is the one with the greatest sum + of weights, i.e. for each node that meets + all of the scheduling requirements (resource + request, requiredDuringScheduling anti-affinity + expressions, etc.), compute a sum by iterating + through the elements of this field and adding + "weight" to the sum if the node has pods which + matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most + preferred. items: - description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most preferred node(s) + description: The weights of all of the matched + WeightedPodAffinityTerm fields are added + per-node to find the most preferred node(s) properties: podAffinityTerm: - description: Required. A pod affinity term, associated with the corresponding weight. + description: Required. A pod affinity + term, associated with the corresponding + weight. properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a + set of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions + is a list of label selector + requirements. The requirements + are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector + requirement is a selector + that contains values, a key, + and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the + label key that the selector + applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to + a set of values. Valid + operators are In, NotIn, + Exists and DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an + array of string values. + If the operator is In + or NotIn, the values array + must be non-empty. If + the operator is Exists + or DoesNotExist, the values + array must be empty. This + array is replaced during + a strategic merge patch. items: type: string type: array @@ -732,22 +1251,45 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a + map of {key,value} pairs. A + single {key,value} in the matchLabels + map is equivalent to an element + of matchExpressions, whose key + field is "key", the operator + is "In", and the values array + contains only "value". The requirements + are ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies + which namespaces the labelSelector + applies to (matches against); null + or empty list means "this pod's + namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where + co-located is defined as running + on a node whose value of the label + with key topologyKey matches that + of any node on which any of the + selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey type: object weight: - description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + description: weight associated with matching + the corresponding podAffinityTerm, in + the range 1-100. format: int32 type: integer required: @@ -756,26 +1298,64 @@ spec: type: object type: array requiredDuringSchedulingIgnoredDuringExecution: - description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod will not be scheduled onto the node. If the anti-affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to a pod label update), the system may or may not try to eventually evict the pod from its node. When there are multiple elements, the lists of nodes corresponding to each podAffinityTerm are intersected, i.e. all terms must be satisfied. + description: If the anti-affinity requirements + specified by this field are not met at scheduling + time, the pod will not be scheduled onto the + node. If the anti-affinity requirements specified + by this field cease to be met at some point + during pod execution (e.g. due to a pod label + update), the system may or may not try to + eventually evict the pod from its node. When + there are multiple elements, the lists of + nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. items: - description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) that this pod should be co-located (affinity) or not co-located (anti-affinity) with, where co-located is defined as running on a node whose value of the label with key matches that of any node on which a pod of the set of pods is running + description: Defines a set of pods (namely + those matching the labelSelector relative + to the given namespace(s)) that this pod + should be co-located (affinity) or not co-located + (anti-affinity) with, where co-located is + defined as running on a node whose value + of the label with key matches + that of any node on which a pod of the set + of pods is running properties: labelSelector: - description: A label query over a set of resources, in this case pods. + description: A label query over a set + of resources, in this case pods. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a + list of label selector requirements. + The requirements are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label + key that the selector applies + to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents + a key's relationship to a + set of values. Valid operators + are In, NotIn, Exists and + DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array + of string values. If the operator + is In or NotIn, the values + array must be non-empty. If + the operator is Exists or + DoesNotExist, the values array + must be empty. This array + is replaced during a strategic + merge patch. items: type: string type: array @@ -787,16 +1367,34 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map + of {key,value} pairs. A single {key,value} + in the matchLabels map is equivalent + to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are + ANDed. type: object type: object namespaces: - description: namespaces specifies which namespaces the labelSelector applies to (matches against); null or empty list means "this pod's namespace" + description: namespaces specifies which + namespaces the labelSelector applies + to (matches against); null or empty + list means "this pod's namespace" items: type: string type: array topologyKey: - description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching the labelSelector in the specified namespaces, where co-located is defined as running on a node whose value of the label with key topologyKey matches that of any node on which any of the selected pods is running. Empty topologyKey is not allowed. + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose + value of the label with key topologyKey + matches that of any node on which any + of the selected pods is running. Empty + topologyKey is not allowed. type: string required: - topologyKey @@ -805,36 +1403,76 @@ spec: type: object type: object automountServiceAccountToken: - description: AutomountServiceAccountToken indicates whether a service account token should be automatically mounted. + description: AutomountServiceAccountToken indicates + whether a service account token should be automatically + mounted. type: boolean containers: - description: List of containers belonging to the pod. Containers cannot currently be added or removed. There must be at least one container in a Pod. Cannot be updated. + description: List of containers belonging to the pod. + Containers cannot currently be added or removed. There + must be at least one container in a Pod. Cannot be + updated. items: - description: A single application container that you want to run within a pod. + description: A single application container that you + want to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The + docker image''s CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable + cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within + a shell. The docker image''s ENTRYPOINT is used + if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a + double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to + set in the container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment + variable present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) + are expanded using the previous defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. The $(VAR_NAME) syntax can + be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment + variable's value. Cannot be used if value + is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -843,56 +1481,84 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the + ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to + select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required + for volumes, optional for env + vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output + format of the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: resource + to select' type: string required: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret + in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret + to select from. Must be a valid + secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the + Secret or its key must be defined type: boolean required: - key @@ -903,72 +1569,127 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source + of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap + must be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret + must be defined type: boolean type: object type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level + config management to default or override container + images in workload controllers like Deployments + and StatefulSets.' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. + description: Actions that the management system + should take in response to container lifecycle + events. Cannot be updated. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -976,58 +1697,108 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The reason for termination is passed to + the handler. The Pod''s termination grace + period countdown begins before the PreStop + hooked is executed. Regardless of the outcome + of the handler, the container will eventually + terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until + the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -1035,25 +1806,38 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -1061,31 +1845,53 @@ spec: type: object type: object livenessProbe: - description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1099,70 +1905,115 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + description: Name of the container specified as + a DNS_LABEL. Each container in a pod must have + a unique name (DNS_LABEL). Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. + description: List of ports to expose from the + container. Exposing a port here gives the system + additional information about the network connections + a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent + that port from being exposed. Any port which + is listening on the default "0.0.0.0" address + inside a container will be accessible from the + network. Cannot be updated. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network + port in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external + port to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort @@ -1174,31 +2025,54 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container service + readiness. Container will be removed from service + endpoints if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1212,48 +2086,71 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Compute Resources required by this + container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -1262,7 +2159,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -1271,107 +2170,213 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'Security options the pod should + run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a + read-only root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label + that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label + that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label + that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label + that applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. type: string type: object type: object startupProbe: - description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'StartupProbe indicates that the + Pod has successfully initialized. If specified, + no other probes are executed until this completes + successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. + This can be used to provide different probe + parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data + or warm a cache, than during steady-state operation. + This cannot be updated. This is a beta feature + enabled by the StartupProbe feature flag. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1385,77 +2390,139 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false type: boolean terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block + devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping + of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside + of the container that the device will + be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of + a persistentVolumeClaim in the pod type: string required: - devicePath @@ -1463,27 +2530,46 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting + of a Volume within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. type: string name: - description: This must match the Name of a Volume. + description: This must match the Name of + a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. type: string required: - mountPath @@ -1491,24 +2577,37 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. type: string required: - name type: object type: array dnsConfig: - description: Specifies the DNS parameters of a pod. Parameters specified here will be merged to the generated DNS configuration based on DNSPolicy. + description: Specifies the DNS parameters of a pod. + Parameters specified here will be merged to the generated + DNS configuration based on DNSPolicy. properties: nameservers: - description: A list of DNS name server IP addresses. This will be appended to the base nameservers generated from DNSPolicy. Duplicated nameservers will be removed. + description: A list of DNS name server IP addresses. + This will be appended to the base nameservers + generated from DNSPolicy. Duplicated nameservers + will be removed. items: type: string type: array options: - description: A list of DNS resolver options. This will be merged with the base options generated from DNSPolicy. Duplicated entries will be removed. Resolution options given in Options will override those that appear in the base DNSPolicy. + description: A list of DNS resolver options. This + will be merged with the base options generated + from DNSPolicy. Duplicated entries will be removed. + Resolution options given in Options will override + those that appear in the base DNSPolicy. items: - description: PodDNSConfigOption defines DNS resolver options of a pod. + description: PodDNSConfigOption defines DNS resolver + options of a pod. properties: name: description: Required. @@ -1518,45 +2617,112 @@ spec: type: object type: array searches: - description: A list of DNS search domains for host-name lookup. This will be appended to the base search paths generated from DNSPolicy. Duplicated search paths will be removed. + description: A list of DNS search domains for host-name + lookup. This will be appended to the base search + paths generated from DNSPolicy. Duplicated search + paths will be removed. items: type: string type: array type: object dnsPolicy: - description: Set DNS policy for the pod. Defaults to "ClusterFirst". Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', 'Default' or 'None'. DNS parameters given in DNSConfig will be merged with the policy selected with DNSPolicy. To have DNS options set along with hostNetwork, you have to specify DNS policy explicitly to 'ClusterFirstWithHostNet'. + description: Set DNS policy for the pod. Defaults to + "ClusterFirst". Valid values are 'ClusterFirstWithHostNet', + 'ClusterFirst', 'Default' or 'None'. DNS parameters + given in DNSConfig will be merged with the policy + selected with DNSPolicy. To have DNS options set along + with hostNetwork, you have to specify DNS policy explicitly + to 'ClusterFirstWithHostNet'. type: string enableServiceLinks: - description: 'EnableServiceLinks indicates whether information about services should be injected into pod''s environment variables, matching the syntax of Docker links. Optional: Defaults to true.' + description: 'EnableServiceLinks indicates whether information + about services should be injected into pod''s environment + variables, matching the syntax of Docker links. Optional: + Defaults to true.' type: boolean ephemeralContainers: - description: List of ephemeral containers run in this pod. Ephemeral containers may be run in an existing pod to perform user-initiated actions such as debugging. This list cannot be specified when creating a pod, and it cannot be modified by updating the pod spec. In order to add an ephemeral container to an existing pod, use the pod's ephemeralcontainers subresource. This field is alpha-level and is only honored by servers that enable the EphemeralContainers feature. + description: List of ephemeral containers run in this + pod. Ephemeral containers may be run in an existing + pod to perform user-initiated actions such as debugging. + This list cannot be specified when creating a pod, + and it cannot be modified by updating the pod spec. + In order to add an ephemeral container to an existing + pod, use the pod's ephemeralcontainers subresource. + This field is alpha-level and is only honored by servers + that enable the EphemeralContainers feature. items: - description: An EphemeralContainer is a container that may be added temporarily to an existing pod for user-initiated activities such as debugging. Ephemeral containers have no resource or scheduling guarantees, and they will not be restarted when they exit or when a pod is removed or restarted. If an ephemeral container causes a pod to exceed its resource allocation, the pod may be evicted. Ephemeral containers may not be added by directly updating the pod spec. They must be added via the pod's ephemeralcontainers subresource, and they will appear in the pod spec once added. This is an alpha feature enabled by the EphemeralContainers feature flag. + description: An EphemeralContainer is a container + that may be added temporarily to an existing pod + for user-initiated activities such as debugging. + Ephemeral containers have no resource or scheduling + guarantees, and they will not be restarted when + they exit or when a pod is removed or restarted. + If an ephemeral container causes a pod to exceed + its resource allocation, the pod may be evicted. + Ephemeral containers may not be added by directly + updating the pod spec. They must be added via the + pod's ephemeralcontainers subresource, and they + will appear in the pod spec once added. This is + an alpha feature enabled by the EphemeralContainers + feature flag. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The + docker image''s CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable + cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within + a shell. The docker image''s ENTRYPOINT is used + if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a + double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to + set in the container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment + variable present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) + are expanded using the previous defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. The $(VAR_NAME) syntax can + be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment + variable's value. Cannot be used if value + is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -1565,56 +2731,84 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the + ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to + select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required + for volumes, optional for env + vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output + format of the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: resource + to select' type: string required: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret + in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret + to select from. Must be a valid + secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the + Secret or its key must be defined type: boolean required: - key @@ -1625,31 +2819,50 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source + of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap + must be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret + must be defined type: boolean type: object type: object @@ -1658,39 +2871,70 @@ spec: description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Lifecycle is not allowed for ephemeral containers. + description: Lifecycle is not allowed for ephemeral + containers. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -1698,64 +2942,114 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The reason for termination is passed to + the handler. The Pod''s termination grace + period countdown begins before the PreStop + hooked is executed. Regardless of the outcome + of the handler, the container will eventually + terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until + the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -1763,31 +3057,44 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -1795,31 +3102,52 @@ spec: type: object type: object livenessProbe: - description: Probes are not allowed for ephemeral containers. + description: Probes are not allowed for ephemeral + containers. properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1833,107 +3161,167 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the ephemeral container specified as a DNS_LABEL. This name must be unique among all containers, init containers and ephemeral containers. + description: Name of the ephemeral container specified + as a DNS_LABEL. This name must be unique among + all containers, init containers and ephemeral + containers. type: string ports: - description: Ports are not allowed for ephemeral containers. + description: Ports are not allowed for ephemeral + containers. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network + port in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external + port to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort type: object type: array readinessProbe: - description: Probes are not allowed for ephemeral containers. + description: Probes are not allowed for ephemeral + containers. properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -1947,54 +3335,78 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: Resources are not allowed for ephemeral containers. Ephemeral containers use spare resources already allocated to the pod. + description: Resources are not allowed for ephemeral + containers. Ephemeral containers use spare resources + already allocated to the pod. properties: limits: additionalProperties: @@ -2003,7 +3415,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -2012,107 +3426,202 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: SecurityContext is not allowed for ephemeral containers. + description: SecurityContext is not allowed for + ephemeral containers. properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a + read-only root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label + that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label + that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label + that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label + that applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. type: string type: object type: object startupProbe: - description: Probes are not allowed for ephemeral containers. + description: Probes are not allowed for ephemeral + containers. properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2126,80 +3635,148 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false type: boolean targetContainerName: - description: If set, the name of the container from PodSpec that this ephemeral container targets. The ephemeral container will be run in the namespaces (IPC, PID, etc) of this container. If not set then the ephemeral container is run in whatever namespaces are shared for the pod. Note that the container runtime must support this feature. + description: If set, the name of the container + from PodSpec that this ephemeral container targets. + The ephemeral container will be run in the namespaces + (IPC, PID, etc) of this container. If not set + then the ephemeral container is run in whatever + namespaces are shared for the pod. Note that + the container runtime must support this feature. type: string terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block + devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping + of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside + of the container that the device will + be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of + a persistentVolumeClaim in the pod type: string required: - devicePath @@ -2207,27 +3784,46 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting + of a Volume within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. type: string name: - description: This must match the Name of a Volume. + description: This must match the Name of + a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. type: string required: - mountPath @@ -2235,16 +3831,24 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. type: string required: - name type: object type: array hostAliases: - description: HostAliases is an optional list of hosts and IPs that will be injected into the pod's hosts file if specified. This is only valid for non-hostNetwork pods. + description: HostAliases is an optional list of hosts + and IPs that will be injected into the pod's hosts + file if specified. This is only valid for non-hostNetwork + pods. items: - description: HostAlias holds the mapping between IP and hostnames that will be injected as an entry in the pod's hosts file. + description: HostAlias holds the mapping between IP + and hostnames that will be injected as an entry + in the pod's hosts file. properties: hostnames: description: Hostnames for the above IP address. @@ -2257,55 +3861,123 @@ spec: type: object type: array hostIPC: - description: 'Use the host''s ipc namespace. Optional: Default to false.' + description: 'Use the host''s ipc namespace. Optional: + Default to false.' type: boolean hostNetwork: - description: Host networking requested for this pod. Use the host's network namespace. If this option is set, the ports that will be used must be specified. Default to false. + description: Host networking requested for this pod. + Use the host's network namespace. If this option is + set, the ports that will be used must be specified. + Default to false. type: boolean hostPID: - description: 'Use the host''s pid namespace. Optional: Default to false.' + description: 'Use the host''s pid namespace. Optional: + Default to false.' type: boolean hostname: - description: Specifies the hostname of the Pod If not specified, the pod's hostname will be set to a system-defined value. + description: Specifies the hostname of the Pod If not + specified, the pod's hostname will be set to a system-defined + value. type: string imagePullSecrets: - description: 'ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec. If specified, these secrets will be passed to individual puller implementations for them to use. For example, in the case of docker, only DockerConfig type secrets are honored. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + description: 'ImagePullSecrets is an optional list of + references to secrets in the same namespace to use + for pulling any of the images used by this PodSpec. + If specified, these secrets will be passed to individual + puller implementations for them to use. For example, + in the case of docker, only DockerConfig type secrets + are honored. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + description: LocalObjectReference contains enough + information to let you locate the referenced object + inside the same namespace. properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' type: string type: object type: array initContainers: - description: 'List of initialization containers belonging to the pod. Init containers are executed in order prior to containers being started. If any init container fails, the pod is considered to have failed and is handled according to its restartPolicy. The name for an init container or normal container must be unique among all containers. Init containers may not have Lifecycle actions, Readiness probes, Liveness probes, or Startup probes. The resourceRequirements of an init container are taken into account during scheduling by finding the highest request/limit for each resource type, and then using the max of of that value or the sum of the normal containers. Limits are applied to init containers in a similar fashion. Init containers cannot currently be added or removed. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' + description: 'List of initialization containers belonging + to the pod. Init containers are executed in order + prior to containers being started. If any init container + fails, the pod is considered to have failed and is + handled according to its restartPolicy. The name for + an init container or normal container must be unique + among all containers. Init containers may not have + Lifecycle actions, Readiness probes, Liveness probes, + or Startup probes. The resourceRequirements of an + init container are taken into account during scheduling + by finding the highest request/limit for each resource + type, and then using the max of of that value or the + sum of the normal containers. Limits are applied to + init containers in a similar fashion. Init containers + cannot currently be added or removed. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/init-containers/' items: - description: A single application container that you want to run within a pod. + description: A single application container that you + want to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The + docker image''s CMD is used if this is not provided. + Variable references $(VAR_NAME) are expanded + using the container''s environment. If a variable + cannot be resolved, the reference in the input + string will be unchanged. The $(VAR_NAME) syntax + can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot + be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within + a shell. The docker image''s ENTRYPOINT is used + if this is not provided. Variable references + $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, + the reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a + double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether + the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to + set in the container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment + variable present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. + Must be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) + are expanded using the previous defined + environment variables in the container + and any service environment variables. + If a variable cannot be resolved, the + reference in the input string will be + unchanged. The $(VAR_NAME) syntax can + be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, + regardless of whether the variable exists + or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment + variable's value. Cannot be used if value + is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -2314,56 +3986,84 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the + ConfigMap or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the + pod: supports metadata.name, metadata.namespace, + metadata.labels, metadata.annotations, + spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema + the FieldPath is written in terms + of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to + select in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required + for volumes, optional for env + vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output + format of the exposed resources, + defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: - description: 'Required: resource to select' + description: 'Required: resource + to select' type: string required: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret + in the pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret + to select from. Must be a valid + secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. + apiVersion, kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the + Secret or its key must be defined type: boolean required: - key @@ -2374,72 +4074,127 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment + variables in the container. The keys defined + within a source must be a C_IDENTIFIER. All + invalid keys will be reported as an event when + the container is starting. When a key exists + in multiple sources, the value associated with + the last source will take precedence. Values + defined by an Env with a duplicate key will + take precedence. Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source + of a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap + must be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend + to each key in the ConfigMap. Must be + a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret + must be defined type: boolean type: object type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level + config management to default or override container + images in workload controllers like Deployments + and StatefulSets.' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, + Never, IfNotPresent. Defaults to Always if :latest + tag is specified, or IfNotPresent otherwise. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. + description: Actions that the management system + should take in response to container lifecycle + events. Cannot be updated. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately + after a container is created. If the handler + fails, the container is terminated and restarted + according to its restart policy. Other management + of the container blocks until the hook completes. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -2447,58 +4202,108 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately + before a container is terminated due to + an API request or management event such + as liveness/startup probe failure, preemption, + resource contention, etc. The handler is + not called if the container crashes or exits. + The reason for termination is passed to + the handler. The Pod''s termination grace + period countdown begins before the PreStop + hooked is executed. Regardless of the outcome + of the handler, the container will eventually + terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until + the termination grace period is reached. + More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies + the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command + line to execute inside the container, + the working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it + is not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to + explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http + request to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect + to, defaults to the pod IP. You + probably want to set "Host" in httpHeaders + instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set + in the request. HTTP allows repeated + headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes + a custom header to be used in + HTTP probes properties: name: - description: The header field name + description: The header field + name type: string value: - description: The header field value + description: The header field + value type: string required: - name @@ -2506,25 +4311,38 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the + HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not + yet supported TODO: implement a realistic + TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name + to connect to, defaults to the pod + IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the + port to access on the container. + Number must be in the range 1 to + 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -2532,31 +4350,53 @@ spec: type: object type: object livenessProbe: - description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container liveness. + Container will be restarted if the probe fails. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2570,70 +4410,115 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + description: Name of the container specified as + a DNS_LABEL. Each container in a pod must have + a unique name (DNS_LABEL). Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. + description: List of ports to expose from the + container. Exposing a port here gives the system + additional information about the network connections + a container uses, but is primarily informational. + Not specifying a port here DOES NOT prevent + that port from being exposed. Any port which + is listening on the default "0.0.0.0" address + inside a container will be accessible from the + network. Cannot be updated. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network + port in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on + the pod's IP address. This must be a valid + port number, 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external + port to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on + the host. If specified, this must be a + valid port number, 0 < x < 65536. If HostNetwork + is specified, this must match ContainerPort. + Most containers do not need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be + an IANA_SVC_NAME and unique within the + pod. Each named port in a pod must have + a unique name. Name for the port that + can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be + UDP, TCP, or SCTP. Defaults to "TCP". type: string required: - containerPort @@ -2645,31 +4530,54 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container service + readiness. Container will be removed from service + endpoints if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2683,48 +4591,71 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Compute Resources required by this + container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -2733,7 +4664,9 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum + amount of compute resources allowed. More + info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -2742,107 +4675,213 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum + amount of compute resources required. If + Requests is omitted for a container, it + defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'Security options the pod should + run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls + whether a process can gain more privileges + than its parent process. This bool directly + controls if the no_new_privs flag will be + set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) + run as Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop + when running containers. Defaults to the + default set of capabilities granted by the + container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX + capabilities type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. + Processes in privileged containers are essentially + equivalent to root on the host. Defaults + to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of + proc mount to use for the containers. The + default is DefaultProcMount which uses the + container runtime defaults for readonly + paths and masked paths. This requires the + ProcMountType feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a + read-only root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint + of the container process. Uses runtime default + if unset. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container + must run as a non-root user. If true, the + Kubelet will validate the image at runtime + to ensure that it does not run as UID 0 + (root) and fail to start the container if + it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint + of the container process. Defaults to user + specified in image metadata if unspecified. + May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied + to the container. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label + that applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label + that applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label + that applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label + that applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings + applied to all containers. If unspecified, + the options from the PodSecurityContext + will be used. If set in both SecurityContext + and PodSecurityContext, the value specified + in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where + the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName + field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is + the name of the GMSA credential spec + to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to + run the entrypoint of the container + process. Defaults to the user specified + in image metadata if unspecified. May + also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext + takes precedence. type: string type: object type: object startupProbe: - description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'StartupProbe indicates that the + Pod has successfully initialized. If specified, + no other probes are executed until this completes + successfully. If this probe fails, the Pod will + be restarted, just as if the livenessProbe failed. + This can be used to provide different probe + parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data + or warm a cache, than during steady-state operation. + This cannot be updated. This is a beta feature + enabled by the StartupProbe feature flag. More + info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the + action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line + to execute inside the container, the + working directory for the command is + root ('/') in the container's filesystem. + The command is simply exec'd, it is + not run inside a shell, so traditional + shell instructions ('|', etc) won't + work. To use a shell, you need to explicitly + call out to that shell. Exit status + of 0 is treated as live/healthy and + non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures + for the probe to be considered failed after + having succeeded. Defaults to 3. Minimum + value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, + defaults to the pod IP. You probably + want to set "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in + the request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a + custom header to be used in HTTP probes properties: name: description: The header field name @@ -2856,77 +4895,139 @@ spec: type: object type: array path: - description: Path to access on the HTTP server. + description: Path to access on the HTTP + server. type: string port: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the + container has started before liveness probes + are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform + the probe. Default to 10 seconds. Minimum + value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes + for the probe to be considered successful + after having failed. Defaults to 1. Must + be 1 for liveness and startup. Minimum value + is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action + involving a TCP port. TCP hooks not yet + supported TODO: implement a realistic TCP + lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port + to access on the container. Number must + be in the range 1 to 65535. Name must + be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which + the probe times out. Defaults to 1 second. + Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate + a buffer for stdin in the container runtime. + If this is not set, reads from stdin in the + container will always result in EOF. Default + is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should + close the stdin channel after it has been opened + by a single attach. When stdin is true the stdin + stream will remain open across multiple attach + sessions. If stdinOnce is set to true, stdin + is opened on container start, is empty until + the first client attaches to stdin, and then + remains open and accepts data until the client + disconnects, at which time stdin is closed and + remains closed until the container is restarted. + If this flag is false, a container processes + that reads from stdin will never receive an + EOF. Default is false type: boolean terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file + to which the container''s termination message + will be written is mounted into the container''s + filesystem. Message written is intended to be + brief final status, such as an assertion failure + message. Will be truncated by the node if greater + than 4096 bytes. The total message length across + all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message + should be populated. File will use the contents + of terminationMessagePath to populate the container + status message on both success and failure. + FallbackToLogsOnError will use the last chunk + of container log output if the termination message + file is empty and the container exited with + an error. The log output is limited to 2048 + bytes or 80 lines, whichever is smaller. Defaults + to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate + a TTY for itself, also requires 'stdin' to be + true. Default is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block + devices to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping + of a raw block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside + of the container that the device will + be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of + a persistentVolumeClaim in the pod type: string required: - devicePath @@ -2934,27 +5035,46 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting + of a Volume within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at + which the volume should be mounted. Must + not contain ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines + how mounts are propagated from the host + to container and the other way around. + When not set, MountPropagationNone is + used. This field is beta in 1.10. type: string name: - description: This must match the Name of a Volume. + description: This must match the Name of + a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, + read-write otherwise (false or unspecified). + Defaults to false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from + which the container's volume should be + mounted. Defaults to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume + from which the container's volume should + be mounted. Behaves similarly to SubPath + but environment variable references $(VAR_NAME) + are expanded using the container's environment. + Defaults to "" (volume's root). SubPathExpr + and SubPath are mutually exclusive. type: string required: - mountPath @@ -2962,19 +5082,28 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If + not specified, the container runtime's default + will be used, which might be configured in the + container image. Cannot be updated. type: string required: - name type: object type: array nodeName: - description: NodeName is a request to schedule this pod onto a specific node. If it is non-empty, the scheduler simply schedules this pod onto that node, assuming that it fits resource requirements. + description: NodeName is a request to schedule this + pod onto a specific node. If it is non-empty, the + scheduler simply schedules this pod onto that node, + assuming that it fits resource requirements. type: string nodeSelector: additionalProperties: type: string - description: 'NodeSelector is a selector which must be true for the pod to fit on a node. Selector which must match a node''s labels for the pod to be scheduled on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + description: 'NodeSelector is a selector which must + be true for the pod to fit on a node. Selector which + must match a node''s labels for the pod to be scheduled + on that node. More info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' type: object overhead: additionalProperties: @@ -2983,86 +5112,185 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Overhead represents the resource overhead associated with running a pod for a given RuntimeClass. This field will be autopopulated at admission time by the RuntimeClass admission controller. If the RuntimeClass admission controller is enabled, overhead must not be set in Pod create requests. The RuntimeClass admission controller will reject Pod create requests which have the overhead already set. If RuntimeClass is configured and selected in the PodSpec, Overhead will be set to the value defined in the corresponding RuntimeClass, otherwise it will remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md This field is alpha-level as of Kubernetes v1.16, and is only honored by servers that enable the PodOverhead feature.' + description: 'Overhead represents the resource overhead + associated with running a pod for a given RuntimeClass. + This field will be autopopulated at admission time + by the RuntimeClass admission controller. If the RuntimeClass + admission controller is enabled, overhead must not + be set in Pod create requests. The RuntimeClass admission + controller will reject Pod create requests which have + the overhead already set. If RuntimeClass is configured + and selected in the PodSpec, Overhead will be set + to the value defined in the corresponding RuntimeClass, + otherwise it will remain unset and treated as zero. + More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md + This field is alpha-level as of Kubernetes v1.16, + and is only honored by servers that enable the PodOverhead + feature.' type: object preemptionPolicy: - description: PreemptionPolicy is the Policy for preempting pods with lower priority. One of Never, PreemptLowerPriority. Defaults to PreemptLowerPriority if unset. This field is alpha-level and is only honored by servers that enable the NonPreemptingPriority feature. + description: PreemptionPolicy is the Policy for preempting + pods with lower priority. One of Never, PreemptLowerPriority. + Defaults to PreemptLowerPriority if unset. This field + is alpha-level and is only honored by servers that + enable the NonPreemptingPriority feature. type: string priority: - description: The priority value. Various system components use this field to find the priority of the pod. When Priority Admission Controller is enabled, it prevents users from setting this field. The admission controller populates this field from PriorityClassName. The higher the value, the higher the priority. + description: The priority value. Various system components + use this field to find the priority of the pod. When + Priority Admission Controller is enabled, it prevents + users from setting this field. The admission controller + populates this field from PriorityClassName. The higher + the value, the higher the priority. format: int32 type: integer priorityClassName: - description: If specified, indicates the pod's priority. "system-node-critical" and "system-cluster-critical" are two special keywords which indicate the highest priorities with the former being the highest priority. Any other name must be defined by creating a PriorityClass object with that name. If not specified, the pod priority will be default or zero if there is no default. + description: If specified, indicates the pod's priority. + "system-node-critical" and "system-cluster-critical" + are two special keywords which indicate the highest + priorities with the former being the highest priority. + Any other name must be defined by creating a PriorityClass + object with that name. If not specified, the pod priority + will be default or zero if there is no default. type: string readinessGates: - description: 'If specified, all readiness gates will be evaluated for pod readiness. A pod is ready when all its containers are ready AND all conditions specified in the readiness gates have status equal to "True" More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md' + description: 'If specified, all readiness gates will + be evaluated for pod readiness. A pod is ready when + all its containers are ready AND all conditions specified + in the readiness gates have status equal to "True" + More info: https://git.k8s.io/enhancements/keps/sig-network/0007-pod-ready%2B%2B.md' items: - description: PodReadinessGate contains the reference to a pod condition + description: PodReadinessGate contains the reference + to a pod condition properties: conditionType: - description: ConditionType refers to a condition in the pod's condition list with matching type. + description: ConditionType refers to a condition + in the pod's condition list with matching type. type: string required: - conditionType type: object type: array restartPolicy: - description: 'Restart policy for all containers within the pod. One of Always, OnFailure, Never. Default to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' + description: 'Restart policy for all containers within + the pod. One of Always, OnFailure, Never. Default + to Always. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' type: string runtimeClassName: - description: 'RuntimeClassName refers to a RuntimeClass object in the node.k8s.io group, which should be used to run this pod. If no RuntimeClass resource matches the named class, the pod will not be run. If unset or empty, the "legacy" RuntimeClass will be used, which is an implicit class with an empty definition that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md This is a beta feature as of Kubernetes v1.14.' + description: 'RuntimeClassName refers to a RuntimeClass + object in the node.k8s.io group, which should be used + to run this pod. If no RuntimeClass resource matches + the named class, the pod will not be run. If unset + or empty, the "legacy" RuntimeClass will be used, + which is an implicit class with an empty definition + that uses the default runtime handler. More info: + https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md + This is a beta feature as of Kubernetes v1.14.' type: string schedulerName: - description: If specified, the pod will be dispatched by specified scheduler. If not specified, the pod will be dispatched by default scheduler. + description: If specified, the pod will be dispatched + by specified scheduler. If not specified, the pod + will be dispatched by default scheduler. type: string securityContext: - description: 'SecurityContext holds pod-level security attributes and common container settings. Optional: Defaults to empty. See type description for default values of each field.' + description: 'SecurityContext holds pod-level security + attributes and common container settings. Optional: + Defaults to empty. See type description for default + values of each field.' properties: fsGroup: - description: "A special supplemental group that applies to all containers in a pod. Some volume types allow the Kubelet to change the ownership of that volume to be owned by the pod: \n 1. The owning GID will be the FSGroup 2. The setgid bit is set (new files created in the volume will be owned by FSGroup) 3. The permission bits are OR'd with rw-rw---- \n If unset, the Kubelet will not modify the ownership and permissions of any volume." + description: "A special supplemental group that + applies to all containers in a pod. Some volume + types allow the Kubelet to change the ownership + of that volume to be owned by the pod: \n 1. The + owning GID will be the FSGroup 2. The setgid bit + is set (new files created in the volume will be + owned by FSGroup) 3. The permission bits are OR'd + with rw-rw---- \n If unset, the Kubelet will not + modify the ownership and permissions of any volume." format: int64 type: integer fsGroupChangePolicy: - description: 'fsGroupChangePolicy defines behavior of changing ownership and permission of the volume before being exposed inside Pod. This field will only apply to volume types which support fsGroup based ownership(and permissions). It will have no effect on ephemeral volume types such as: secret, configmaps and emptydir. Valid values are "OnRootMismatch" and "Always". If not specified defaults to "Always".' + description: 'fsGroupChangePolicy defines behavior + of changing ownership and permission of the volume + before being exposed inside Pod. This field will + only apply to volume types which support fsGroup + based ownership(and permissions). It will have + no effect on ephemeral volume types such as: secret, + configmaps and emptydir. Valid values are "OnRootMismatch" + and "Always". If not specified defaults to "Always".' type: string runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in SecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence + for that container. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will + validate the image at runtime to ensure that it + does not run as UID 0 (root) and fail to start + the container if it does. If unset or false, no + such validation will be performed. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + description: The UID to run the entrypoint of the + container process. Defaults to user specified + in image metadata if unspecified. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in + SecurityContext takes precedence for that container. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to all containers. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in SecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence for that container. + description: The SELinux context to be applied to + all containers. If unspecified, the container + runtime will allocate a random SELinux context + for each container. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence + for that container. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that + applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that + applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that + applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that + applies to the container. type: string type: object supplementalGroups: - description: A list of groups applied to the first process run in each container, in addition to the container's primary GID. If unspecified, no groups will be added to any container. + description: A list of groups applied to the first + process run in each container, in addition to + the container's primary GID. If unspecified, + no groups will be added to any container. items: format: int64 type: integer type: array sysctls: - description: Sysctls hold a list of namespaced sysctls used for the pod. Pods with unsupported sysctls (by the container runtime) might fail to launch. + description: Sysctls hold a list of namespaced sysctls + used for the pod. Pods with unsupported sysctls + (by the container runtime) might fail to launch. items: - description: Sysctl defines a kernel parameter to be set + description: Sysctl defines a kernel parameter + to be set properties: name: description: Name of a property to set @@ -3076,79 +5304,161 @@ spec: type: object type: array windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options within a container's SecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings applied + to all containers. If unspecified, the options + within a container's SecurityContext will be used. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where the + GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to run + the entrypoint of the container process. Defaults + to the user specified in image metadata if + unspecified. May also be set in PodSecurityContext. + If set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: string type: object type: object serviceAccount: - description: 'DeprecatedServiceAccount is a depreciated alias for ServiceAccountName. Deprecated: Use serviceAccountName instead.' + description: 'DeprecatedServiceAccount is a depreciated + alias for ServiceAccountName. Deprecated: Use serviceAccountName + instead.' type: string serviceAccountName: - description: 'ServiceAccountName is the name of the ServiceAccount to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + description: 'ServiceAccountName is the name of the + ServiceAccount to use to run this pod. More info: + https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' type: string shareProcessNamespace: - description: 'Share a single process namespace between all of the containers in a pod. When this is set containers will be able to view and signal processes from other containers in the same pod, and the first process in each container will not be assigned PID 1. HostPID and ShareProcessNamespace cannot both be set. Optional: Default to false.' + description: 'Share a single process namespace between + all of the containers in a pod. When this is set containers + will be able to view and signal processes from other + containers in the same pod, and the first process + in each container will not be assigned PID 1. HostPID + and ShareProcessNamespace cannot both be set. Optional: + Default to false.' type: boolean subdomain: - description: If specified, the fully qualified Pod hostname will be "...svc.". If not specified, the pod will not have a domainname at all. + description: If specified, the fully qualified Pod hostname + will be "...svc.". If not specified, the pod will not have + a domainname at all. type: string terminationGracePeriodSeconds: - description: Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates delete immediately. If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds. + description: Optional duration in seconds the pod needs + to terminate gracefully. May be decreased in delete + request. Value must be non-negative integer. The value + zero indicates delete immediately. If this value is + nil, the default grace period will be used instead. + The grace period is the duration in seconds after + the processes running in the pod are sent a termination + signal and the time when the processes are forcibly + halted with a kill signal. Set this value longer than + the expected cleanup time for your process. Defaults + to 30 seconds. format: int64 type: integer tolerations: description: If specified, the pod's tolerations. items: - description: The pod this Toleration is attached to tolerates any taint that matches the triple using the matching operator . + description: The pod this Toleration is attached to + tolerates any taint that matches the triple + using the matching operator . properties: effect: - description: Effect indicates the taint effect to match. Empty means match all taint effects. When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + description: Effect indicates the taint effect + to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, + PreferNoSchedule and NoExecute. type: string key: - description: Key is the taint key that the toleration applies to. Empty means match all taint keys. If the key is empty, operator must be Exists; this combination means to match all values and all keys. + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; + this combination means to match all values and + all keys. type: string operator: - description: Operator represents a key's relationship to the value. Valid operators are Exists and Equal. Defaults to Equal. Exists is equivalent to wildcard for value, so that a pod can tolerate all taints of a particular category. + description: Operator represents a key's relationship + to the value. Valid operators are Exists and + Equal. Defaults to Equal. Exists is equivalent + to wildcard for value, so that a pod can tolerate + all taints of a particular category. type: string tolerationSeconds: - description: TolerationSeconds represents the period of time the toleration (which must be of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, it is not set, which means tolerate the taint forever (do not evict). Zero and negative values will be treated as 0 (evict immediately) by the system. + description: TolerationSeconds represents the + period of time the toleration (which must be + of effect NoExecute, otherwise this field is + ignored) tolerates the taint. By default, it + is not set, which means tolerate the taint forever + (do not evict). Zero and negative values will + be treated as 0 (evict immediately) by the system. format: int64 type: integer value: - description: Value is the taint value the toleration matches to. If the operator is Exists, the value should be empty, otherwise just a regular string. + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value + should be empty, otherwise just a regular string. type: string type: object type: array topologySpreadConstraints: - description: TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. This field is only honored by clusters that enable the EvenPodsSpread feature. All topologySpreadConstraints are ANDed. + description: TopologySpreadConstraints describes how + a group of pods ought to spread across topology domains. + Scheduler will schedule pods in a way which abides + by the constraints. This field is only honored by + clusters that enable the EvenPodsSpread feature. All + topologySpreadConstraints are ANDed. items: - description: TopologySpreadConstraint specifies how to spread matching pods among the given topology. + description: TopologySpreadConstraint specifies how + to spread matching pods among the given topology. properties: labelSelector: - description: LabelSelector is used to find matching pods. Pods that match this label selector are counted to determine the number of pods in their corresponding topology domain. + description: LabelSelector is used to find matching + pods. Pods that match this label selector are + counted to determine the number of pods in their + corresponding topology domain. properties: matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. properties: key: - description: key is the label key that the selector applies to. + description: key is the label key that + the selector applies to. type: string operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. type: string values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. items: type: string type: array @@ -3160,18 +5470,58 @@ spec: matchLabels: additionalProperties: type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. type: object type: object maxSkew: - description: 'MaxSkew describes the degree to which pods may be unevenly distributed. It''s the maximum permitted difference between the number of matching pods in any two topology domains of a given topology type. For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 1/1/0: | zone1 | zone2 | zone3 | | P | P | | - if MaxSkew is 1, incoming pod can only be scheduled to zone3 to become 1/1/1; scheduling it onto zone1(zone2) would make the ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). - if MaxSkew is 2, incoming pod can be scheduled onto any zone. It''s a required field. Default value is 1 and 0 is not allowed.' + description: 'MaxSkew describes the degree to + which pods may be unevenly distributed. It''s + the maximum permitted difference between the + number of matching pods in any two topology + domains of a given topology type. For example, + in a 3-zone cluster, MaxSkew is set to 1, and + pods with the same labelSelector spread as 1/1/0: + | zone1 | zone2 | zone3 | | P | P | | + - if MaxSkew is 1, incoming pod can only be + scheduled to zone3 to become 1/1/1; scheduling + it onto zone1(zone2) would make the ActualSkew(2-0) + on zone1(zone2) violate MaxSkew(1). - if MaxSkew + is 2, incoming pod can be scheduled onto any + zone. It''s a required field. Default value + is 1 and 0 is not allowed.' format: int32 type: integer topologyKey: - description: TopologyKey is the key of node labels. Nodes that have a label with this key and identical values are considered to be in the same topology. We consider each as a "bucket", and try to put balanced number of pods into each bucket. It's a required field. + description: TopologyKey is the key of node labels. + Nodes that have a label with this key and identical + values are considered to be in the same topology. + We consider each as a "bucket", + and try to put balanced number of pods into + each bucket. It's a required field. type: string whenUnsatisfiable: - description: 'WhenUnsatisfiable indicates how to deal with a pod if it doesn''t satisfy the spread constraint. - DoNotSchedule (default) tells the scheduler not to schedule it - ScheduleAnyway tells the scheduler to still schedule it It''s considered as "Unsatisfiable" if and only if placing incoming pod on any topology violates "MaxSkew". For example, in a 3-zone cluster, MaxSkew is set to 1, and pods with the same labelSelector spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | If WhenUnsatisfiable is set to DoNotSchedule, incoming pod can only be scheduled to zone2(zone3) to become 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies MaxSkew(1). In other words, the cluster can still be imbalanced, but scheduler won''t make it *more* imbalanced. It''s a required field.' + description: 'WhenUnsatisfiable indicates how + to deal with a pod if it doesn''t satisfy the + spread constraint. - DoNotSchedule (default) + tells the scheduler not to schedule it - ScheduleAnyway + tells the scheduler to still schedule it It''s + considered as "Unsatisfiable" if and only if + placing incoming pod on any topology violates + "MaxSkew". For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same + labelSelector spread as 3/1/1: | zone1 | zone2 + | zone3 | | P P P | P | P | If WhenUnsatisfiable + is set to DoNotSchedule, incoming pod can only + be scheduled to zone2(zone3) to become 3/2/1(3/1/2) + as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can + still be imbalanced, but scheduler won''t make + it *more* imbalanced. It''s a required field.' type: string required: - maxSkew @@ -3184,7 +5534,8 @@ spec: - whenUnsatisfiable x-kubernetes-list-type: map volumes: - description: 'List of volumes that can be mounted by containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' + description: 'List of volumes that can be mounted by + containers belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' items: type: object type: array @@ -3194,7 +5545,8 @@ spec: type: object type: array engineResources: - description: ResourceRequirements describes the compute resource requirements. + description: ResourceRequirements describes the compute resource + requirements. properties: limits: additionalProperties: @@ -3203,7 +5555,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -3212,7 +5565,10 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object explainer: @@ -3222,31 +5578,60 @@ spec: type: string type: object containerSpec: - description: A single application container that you want to run within a pod. + description: A single application container that you want + to run within a pod. properties: args: - description: 'Arguments to the entrypoint. The docker image''s CMD is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Arguments to the entrypoint. The docker + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array command: - description: 'Entrypoint array. Not executed within a shell. The docker image''s ENTRYPOINT is used if this is not provided. Variable references $(VAR_NAME) are expanded using the container''s environment. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + description: 'Entrypoint array. Not executed within a + shell. The docker image''s ENTRYPOINT is used if this + is not provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string + will be unchanged. The $(VAR_NAME) syntax can be escaped + with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' items: type: string type: array env: - description: List of environment variables to set in the container. Cannot be updated. + description: List of environment variables to set in the + container. Cannot be updated. items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable + present in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. Must + be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) are + expanded using the previous defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will + never be expanded, regardless of whether the variable + exists or not. Defaults to "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's + value. Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -3255,37 +5640,53 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap + or its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select + in the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for + volumes, optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format + of the exposed resources, defaults to + "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -3295,16 +5696,22 @@ spec: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret in the + pod's namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select + from. Must be a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret + or its key must be defined type: boolean required: - key @@ -3315,66 +5722,111 @@ spec: type: object type: array envFrom: - description: List of sources to populate environment variables in the container. The keys defined within a source must be a C_IDENTIFIER. All invalid keys will be reported as an event when the container is starting. When a key exists in multiple sources, the value associated with the last source will take precedence. Values defined by an Env with a duplicate key will take precedence. Cannot be updated. + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported + as an event when the container is starting. When a key + exists in multiple sources, the value associated with + the last source will take precedence. Values defined + by an Env with a duplicate key will take precedence. + Cannot be updated. items: - description: EnvFromSource represents the source of a set of ConfigMaps + description: EnvFromSource represents the source of + a set of ConfigMaps properties: configMapRef: description: The ConfigMap to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap must be defined + description: Specify whether the ConfigMap must + be defined type: boolean type: object prefix: - description: An optional identifier to prepend to each key in the ConfigMap. Must be a C_IDENTIFIER. + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. type: string secretRef: description: The Secret to select from properties: name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret must be defined + description: Specify whether the Secret must + be defined type: boolean type: object type: object type: array image: - description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images This field is optional to allow higher level config management to default or override container images in workload controllers like Deployments and StatefulSets.' + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images in + workload controllers like Deployments and StatefulSets.' type: string imagePullPolicy: - description: 'Image pull policy. One of Always, Never, IfNotPresent. Defaults to Always if :latest tag is specified, or IfNotPresent otherwise. Cannot be updated. More info: https://kubernetes.io/docs/concepts/containers/images#updating-images' + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is specified, + or IfNotPresent otherwise. Cannot be updated. More info: + https://kubernetes.io/docs/concepts/containers/images#updating-images' type: string lifecycle: - description: Actions that the management system should take in response to container lifecycle events. Cannot be updated. + description: Actions that the management system should + take in response to container lifecycle events. Cannot + be updated. properties: postStart: - description: 'PostStart is called immediately after a container is created. If the handler fails, the container is terminated and restarted according to its restart policy. Other management of the container blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according + to its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the action + to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the + request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom + header to be used in HTTP probes properties: name: description: The header field name @@ -3391,49 +5843,87 @@ spec: description: Path to access on the HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object type: object preStop: - description: 'PreStop is called immediately before a container is terminated due to an API request or management event such as liveness/startup probe failure, preemption, resource contention, etc. The handler is not called if the container crashes or exits. The reason for termination is passed to the handler. The Pod''s termination grace period countdown begins before the PreStop hooked is executed. Regardless of the outcome of the handler, the container will eventually terminate within the Pod''s termination grace period. Other management of the container blocks until the hook completes or until the termination grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup probe + failure, preemption, resource contention, etc. The + handler is not called if the container crashes or + exits. The reason for termination is passed to the + handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will + eventually terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following + should be specified. Exec specifies the action + to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request + to perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the + request. HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom + header to be used in HTTP probes properties: name: description: The header field name @@ -3450,22 +5940,32 @@ spec: description: Path to access on the HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting + to the host. Defaults to HTTP. type: string required: - port type: object tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect + to, defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port @@ -3473,31 +5973,49 @@ spec: type: object type: object livenessProbe: - description: 'Periodic probe of container liveness. Container will be restarted if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container liveness. Container + will be restarted if the probe fails. Cannot be updated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following should + be specified. Exec specifies the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request to + perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the request. + HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom header + to be used in HTTP probes properties: name: description: The header field name @@ -3514,67 +6032,104 @@ spec: description: Path to access on the HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting to the + host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. + Minimum value is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect to, + defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object name: - description: Name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. type: string ports: - description: List of ports to expose from the container. Exposing a port here gives the system additional information about the network connections a container uses, but is primarily informational. Not specifying a port here DOES NOT prevent that port from being exposed. Any port which is listening on the default "0.0.0.0" address inside a container will be accessible from the network. Cannot be updated. + description: List of ports to expose from the container. + Exposing a port here gives the system additional information + about the network connections a container uses, but + is primarily informational. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port + which is listening on the default "0.0.0.0" address + inside a container will be accessible from the network. + Cannot be updated. items: - description: ContainerPort represents a network port in a single container. + description: ContainerPort represents a network port + in a single container. properties: containerPort: - description: Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. format: int32 type: integer hostIP: - description: What host IP to bind the external port to. + description: What host IP to bind the external port + to. type: string hostPort: - description: Number of port to expose on the host. If specified, this must be a valid port number, 0 < x < 65536. If HostNetwork is specified, this must match ContainerPort. Most containers do not need this. + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. format: int32 type: integer name: - description: If specified, this must be an IANA_SVC_NAME and unique within the pod. Each named port in a pod must have a unique name. Name for the port that can be referred to by services. + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the port + that can be referred to by services. type: string protocol: - description: Protocol for port. Must be UDP, TCP, or SCTP. Defaults to "TCP". + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". type: string required: - containerPort @@ -3586,31 +6141,49 @@ spec: - protocol x-kubernetes-list-type: map readinessProbe: - description: 'Periodic probe of container service readiness. Container will be removed from service endpoints if the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Periodic probe of container service readiness. + Container will be removed from service endpoints if + the probe fails. Cannot be updated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following should + be specified. Exec specifies the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request to + perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the request. + HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom header + to be used in HTTP probes properties: name: description: The header field name @@ -3627,45 +6200,62 @@ spec: description: Path to access on the HTTP server. type: string port: - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting to the + host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. + Minimum value is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect to, + defaults to the pod IP.' type: string port: - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object resources: - description: 'Compute Resources required by this container. Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' properties: limits: additionalProperties: @@ -3674,7 +6264,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -3683,107 +6274,193 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object securityContext: - description: 'Security options the pod should run with. More info: https://kubernetes.io/docs/concepts/policy/security-context/ More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + description: 'Security options the pod should run with. + More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' properties: allowPrivilegeEscalation: - description: 'AllowPrivilegeEscalation controls whether a process can gain more privileges than its parent process. This bool directly controls if the no_new_privs flag will be set on the container process. AllowPrivilegeEscalation is true always when the container is: 1) run as Privileged 2) has CAP_SYS_ADMIN' + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as + Privileged 2) has CAP_SYS_ADMIN' type: boolean capabilities: - description: The capabilities to add/drop when running containers. Defaults to the default set of capabilities granted by the container runtime. + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. properties: add: description: Added capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array drop: description: Removed capabilities items: - description: Capability represent POSIX capabilities type + description: Capability represent POSIX capabilities + type type: string type: array type: object privileged: - description: Run container in privileged mode. Processes in privileged containers are essentially equivalent to root on the host. Defaults to false. + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. type: boolean procMount: - description: procMount denotes the type of proc mount to use for the containers. The default is DefaultProcMount which uses the container runtime defaults for readonly paths and masked paths. This requires the ProcMountType feature flag to be enabled. + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. type: string readOnlyRootFilesystem: - description: Whether this container has a read-only root filesystem. Default is false. + description: Whether this container has a read-only + root filesystem. Default is false. type: boolean runAsGroup: - description: The GID to run the entrypoint of the container process. Uses runtime default if unset. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. format: int64 type: integer runAsNonRoot: - description: Indicates that the container must run as a non-root user. If true, the Kubelet will validate the image at runtime to ensure that it does not run as UID 0 (root) and fail to start the container if it does. If unset or false, no such validation will be performed. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not + run as UID 0 (root) and fail to start the container + if it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. type: boolean runAsUser: - description: The UID to run the entrypoint of the container process. Defaults to user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UID to run the entrypoint of the + container process. Defaults to user specified in + image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. format: int64 type: integer seLinuxOptions: - description: The SELinux context to be applied to the container. If unspecified, the container runtime will allocate a random SELinux context for each container. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. properties: level: - description: Level is SELinux level label that applies to the container. + description: Level is SELinux level label that + applies to the container. type: string role: - description: Role is a SELinux role label that applies to the container. + description: Role is a SELinux role label that + applies to the container. type: string type: - description: Type is a SELinux type label that applies to the container. + description: Type is a SELinux type label that + applies to the container. type: string user: - description: User is a SELinux user label that applies to the container. + description: User is a SELinux user label that + applies to the container. type: string type: object windowsOptions: - description: The Windows specific settings applied to all containers. If unspecified, the options from the PodSecurityContext will be used. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The Windows specific settings applied + to all containers. If unspecified, the options from + the PodSecurityContext will be used. If set in both + SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. properties: gmsaCredentialSpec: - description: GMSACredentialSpec is where the GMSA admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) inlines the contents of the GMSA credential spec named by the GMSACredentialSpecName field. + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. type: string gmsaCredentialSpecName: - description: GMSACredentialSpecName is the name of the GMSA credential spec to use. + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. type: string runAsUserName: - description: The UserName in Windows to run the entrypoint of the container process. Defaults to the user specified in image metadata if unspecified. May also be set in PodSecurityContext. If set in both SecurityContext and PodSecurityContext, the value specified in SecurityContext takes precedence. + description: The UserName in Windows to run the + entrypoint of the container process. Defaults + to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. type: string type: object type: object startupProbe: - description: 'StartupProbe indicates that the Pod has successfully initialized. If specified, no other probes are executed until this completes successfully. If this probe fails, the Pod will be restarted, just as if the livenessProbe failed. This can be used to provide different probe parameters at the beginning of a Pod''s lifecycle, when it might take a long time to load data or warm a cache, than during steady-state operation. This cannot be updated. This is a beta feature enabled by the StartupProbe feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'StartupProbe indicates that the Pod has + successfully initialized. If specified, no other probes + are executed until this completes successfully. If this + probe fails, the Pod will be restarted, just as if the + livenessProbe failed. This can be used to provide different + probe parameters at the beginning of a Pod''s lifecycle, + when it might take a long time to load data or warm + a cache, than during steady-state operation. This cannot + be updated. This is a beta feature enabled by the StartupProbe + feature flag. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' properties: exec: - description: One and only one of the following should be specified. Exec specifies the action to take. + description: One and only one of the following should + be specified. Exec specifies the action to take. properties: command: - description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + description: Command is the command line to execute + inside the container, the working directory + for the command is root ('/') in the container's + filesystem. The command is simply exec'd, it + is not run inside a shell, so traditional shell + instructions ('|', etc) won't work. To use a + shell, you need to explicitly call out to that + shell. Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. items: type: string type: array type: object failureThreshold: - description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + description: Minimum consecutive failures for the + probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. format: int32 type: integer httpGet: - description: HTTPGet specifies the http request to perform. + description: HTTPGet specifies the http request to + perform. properties: host: - description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + description: Host name to connect to, defaults + to the pod IP. You probably want to set "Host" + in httpHeaders instead. type: string httpHeaders: - description: Custom headers to set in the request. HTTP allows repeated headers. + description: Custom headers to set in the request. + HTTP allows repeated headers. items: - description: HTTPHeader describes a custom header to be used in HTTP probes + description: HTTPHeader describes a custom header + to be used in HTTP probes properties: name: description: The header field name @@ -3803,71 +6480,120 @@ spec: anyOf: - type: integer - type: string - description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Name or number of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true scheme: - description: Scheme to use for connecting to the host. Defaults to HTTP. + description: Scheme to use for connecting to the + host. Defaults to HTTP. type: string required: - port type: object initialDelaySeconds: - description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after the container + has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer periodSeconds: - description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + description: How often (in seconds) to perform the + probe. Default to 10 seconds. Minimum value is 1. format: int32 type: integer successThreshold: - description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + description: Minimum consecutive successes for the + probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. + Minimum value is 1. format: int32 type: integer tcpSocket: - description: 'TCPSocket specifies an action involving a TCP port. TCP hooks not yet supported TODO: implement a realistic TCP lifecycle hook' + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: implement + a realistic TCP lifecycle hook' properties: host: - description: 'Optional: Host name to connect to, defaults to the pod IP.' + description: 'Optional: Host name to connect to, + defaults to the pod IP.' type: string port: anyOf: - type: integer - type: string - description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + description: Number or name of the port to access + on the container. Number must be in the range + 1 to 65535. Name must be an IANA_SVC_NAME. x-kubernetes-int-or-string: true required: - port type: object timeoutSeconds: - description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + description: 'Number of seconds after which the probe + times out. Defaults to 1 second. Minimum value is + 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' format: int32 type: integer type: object stdin: - description: Whether this container should allocate a buffer for stdin in the container runtime. If this is not set, reads from stdin in the container will always result in EOF. Default is false. + description: Whether this container should allocate a + buffer for stdin in the container runtime. If this is + not set, reads from stdin in the container will always + result in EOF. Default is false. type: boolean stdinOnce: - description: Whether the container runtime should close the stdin channel after it has been opened by a single attach. When stdin is true the stdin stream will remain open across multiple attach sessions. If stdinOnce is set to true, stdin is opened on container start, is empty until the first client attaches to stdin, and then remains open and accepts data until the client disconnects, at which time stdin is closed and remains closed until the container is restarted. If this flag is false, a container processes that reads from stdin will never receive an EOF. Default is false + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is + empty until the first client attaches to stdin, and + then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false type: boolean terminationMessagePath: - description: 'Optional: Path at which the file to which the container''s termination message will be written is mounted into the container''s filesystem. Message written is intended to be brief final status, such as an assertion failure message. Will be truncated by the node if greater than 4096 bytes. The total message length across all containers will be limited to 12kb. Defaults to /dev/termination-log. Cannot be updated.' + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such as + an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length + across all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' type: string terminationMessagePolicy: - description: Indicate how the termination message should be populated. File will use the contents of terminationMessagePath to populate the container status message on both success and failure. FallbackToLogsOnError will use the last chunk of container log output if the termination message file is empty and the container exited with an error. The log output is limited to 2048 bytes or 80 lines, whichever is smaller. Defaults to File. Cannot be updated. + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be updated. type: string tty: - description: Whether this container should allocate a TTY for itself, also requires 'stdin' to be true. Default is false. + description: Whether this container should allocate a + TTY for itself, also requires 'stdin' to be true. Default + is false. type: boolean volumeDevices: - description: volumeDevices is the list of block devices to be used by the container. + description: volumeDevices is the list of block devices + to be used by the container. items: - description: volumeDevice describes a mapping of a raw block device within a container. + description: volumeDevice describes a mapping of a raw + block device within a container. properties: devicePath: - description: devicePath is the path inside of the container that the device will be mapped to. + description: devicePath is the path inside of the + container that the device will be mapped to. type: string name: - description: name must match the name of a persistentVolumeClaim in the pod + description: name must match the name of a persistentVolumeClaim + in the pod type: string required: - devicePath @@ -3875,27 +6601,43 @@ spec: type: object type: array volumeMounts: - description: Pod volumes to mount into the container's filesystem. Cannot be updated. + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. items: - description: VolumeMount describes a mounting of a Volume within a container. + description: VolumeMount describes a mounting of a Volume + within a container. properties: mountPath: - description: Path within the container at which the volume should be mounted. Must not contain ':'. + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. type: string mountPropagation: - description: mountPropagation determines how mounts are propagated from the host to container and the other way around. When not set, MountPropagationNone is used. This field is beta in 1.10. + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. type: string name: description: This must match the Name of a Volume. type: string readOnly: - description: Mounted read-only if true, read-write otherwise (false or unspecified). Defaults to false. + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. type: boolean subPath: - description: Path within the volume from which the container's volume should be mounted. Defaults to "" (volume's root). + description: Path within the volume from which the + container's volume should be mounted. Defaults + to "" (volume's root). type: string subPathExpr: - description: Expanded path within the volume from which the container's volume should be mounted. Behaves similarly to SubPath but environment variable references $(VAR_NAME) are expanded using the container's environment. Defaults to "" (volume's root). SubPathExpr and SubPath are mutually exclusive. + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the + container's environment. Defaults to "" (volume's + root). SubPathExpr and SubPath are mutually exclusive. type: string required: - mountPath @@ -3903,13 +6645,22 @@ spec: type: object type: array workingDir: - description: Container's working directory. If not specified, the container runtime's default will be used, which might be configured in the container image. Cannot be updated. + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot be + updated. type: string required: - name type: object endpoint: properties: + grpc_port: + format: int32 + type: integer + http_port: + format: int32 + type: integer service_host: type: string service_port: @@ -3956,13 +6707,18 @@ spec: implementation: type: string logger: - description: Request/response payload logging. v2alpha1 feature that is added to v1 for backwards compatibility while v1 is the storage version. + description: Request/response payload + logging. v2alpha1 feature that is + added to v1 for backwards compatibility + while v1 is the storage version. properties: mode: - description: What payloads to log + description: What payloads to + log type: string url: - description: URL to send request logging CloudEvents + description: URL to send request + logging CloudEvents type: string type: object methods: @@ -4005,13 +6761,17 @@ spec: implementation: type: string logger: - description: Request/response payload logging. v2alpha1 feature that is added to v1 for backwards compatibility while v1 is the storage version. + description: Request/response payload logging. + v2alpha1 feature that is added to v1 for + backwards compatibility while v1 is the + storage version. properties: mode: description: What payloads to log type: string url: - description: URL to send request logging CloudEvents + description: URL to send request logging + CloudEvents type: string type: object methods: @@ -4054,7 +6814,9 @@ spec: implementation: type: string logger: - description: Request/response payload logging. v2alpha1 feature that is added to v1 for backwards compatibility while v1 is the storage version. + description: Request/response payload logging. + v2alpha1 feature that is added to v1 for backwards + compatibility while v1 is the storage version. properties: mode: description: What payloads to log @@ -4103,7 +6865,9 @@ spec: implementation: type: string logger: - description: Request/response payload logging. v2alpha1 feature that is added to v1 for backwards compatibility while v1 is the storage version. + description: Request/response payload logging. v2alpha1 + feature that is added to v1 for backwards compatibility + while v1 is the storage version. properties: mode: description: What payloads to log @@ -4139,6 +6903,12 @@ spec: type: array endpoint: properties: + grpc_port: + format: int32 + type: integer + http_port: + format: int32 + type: integer service_host: type: string service_port: @@ -4152,7 +6922,9 @@ spec: implementation: type: string logger: - description: Request/response payload logging. v2alpha1 feature that is added to v1 for backwards compatibility while v1 is the storage version. + description: Request/response payload logging. v2alpha1 feature + that is added to v1 for backwards compatibility while v1 + is the storage version. properties: mode: description: What payloads to log @@ -4211,16 +6983,27 @@ spec: properties: env: items: - description: EnvVar represents an environment variable present in a Container. + description: EnvVar represents an environment variable present + in a Container. properties: name: - description: Name of the environment variable. Must be a C_IDENTIFIER. + description: Name of the environment variable. Must + be a C_IDENTIFIER. type: string value: - description: 'Variable references $(VAR_NAME) are expanded using the previous defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. The $(VAR_NAME) syntax can be escaped with a double $$, ie: $$(VAR_NAME). Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + description: 'Variable references $(VAR_NAME) are expanded + using the previous defined environment variables in + the container and any service environment variables. + If a variable cannot be resolved, the reference in + the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Defaults to + "".' type: string valueFrom: - description: Source for the environment variable's value. Cannot be used if value is not empty. + description: Source for the environment variable's value. + Cannot be used if value is not empty. properties: configMapKeyRef: description: Selects a key of a ConfigMap. @@ -4229,37 +7012,52 @@ spec: description: The key to select. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the ConfigMap or its key must be defined + description: Specify whether the ConfigMap or + its key must be defined type: boolean required: - key type: object fieldRef: - description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, metadata.labels, metadata.annotations, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' properties: apiVersion: - description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". type: string fieldPath: - description: Path of the field to select in the specified API version. + description: Path of the field to select in + the specified API version. type: string required: - fieldPath type: object resourceFieldRef: - description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' properties: containerName: - description: 'Container name: required for volumes, optional for env vars' + description: 'Container name: required for volumes, + optional for env vars' type: string divisor: anyOf: - type: integer - type: string - description: Specifies the output format of the exposed resources, defaults to "1" + description: Specifies the output format of + the exposed resources, defaults to "1" pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true resource: @@ -4269,16 +7067,22 @@ spec: - resource type: object secretKeyRef: - description: Selects a key of a secret in the pod's namespace + description: Selects a key of a secret in the pod's + namespace properties: key: - description: The key of the secret to select from. Must be a valid secret key. + description: The key of the secret to select + from. Must be a valid secret key. type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?' + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' type: string optional: - description: Specify whether the Secret or its key must be defined + description: Specify whether the Secret or its + key must be defined type: boolean required: - key @@ -4292,7 +7096,8 @@ spec: format: int32 type: integer resources: - description: ResourceRequirements describes the compute resource requirements. + description: ResourceRequirements describes the compute resource + requirements. properties: limits: additionalProperties: @@ -4301,7 +7106,8 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object requests: additionalProperties: @@ -4310,7 +7116,11 @@ spec: - type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true - description: 'Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object type: object diff --git a/python/licenses/license.txt b/python/licenses/license.txt index 94bf915f7f..62e3a8f62f 100644 --- a/python/licenses/license.txt +++ b/python/licenses/license.txt @@ -147,7 +147,7 @@ SOFTWARE. Markdown -3.3.2 +3.3.3 BSD License Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later) Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) @@ -272,7 +272,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. absl-py -0.10.0 +0.11.0 Apache Software License Apache License @@ -702,7 +702,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. google-auth -1.22.1 +1.23.0 Apache Software License Apache License Version 2.0, January 2004 @@ -908,11 +908,11 @@ Apache Software License google-auth-oauthlib -0.4.1 +0.4.2 Apache Software License - Apache License + Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -1104,7 +1104,7 @@ Apache Software License you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -1320,7 +1320,7 @@ Apache Software License grpcio -1.32.0 +1.33.2 Apache Software License Apache License @@ -1763,37 +1763,7 @@ OTHER DEALINGS IN THE SOFTWARE. h5py 2.10.0 BSD License -Copyright (c) 2008 Andrew Collette and contributors -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the - distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +UNKNOWN idna 2.10 @@ -1934,7 +1904,7 @@ THE SOFTWARE. numpy -1.19.2 +1.19.4 BSD Copyright (c) 2005-2020, NumPy Developers. All rights reserved. @@ -1972,8 +1942,99 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. This binary distribution of NumPy also bundles the following software: +Name: OpenBLAS +Files: .libs/libopenb*.so +Description: bundled as a dynamically linked library +Availability: https://github.com/xianyi/OpenBLAS/ +License: 3-clause BSD + Copyright (c) 2011-2014, The OpenBLAS Project + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. Neither the name of the OpenBLAS project nor the names of + its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Name: LAPACK +Files: .libs/libopenb*.so +Description: bundled in OpenBLAS +Availability: https://github.com/xianyi/OpenBLAS/ +License 3-clause BSD + Copyright (c) 1992-2013 The University of Tennessee and The University + of Tennessee Research Foundation. All rights + reserved. + Copyright (c) 2000-2013 The University of California Berkeley. All + rights reserved. + Copyright (c) 2006-2013 The University of Colorado Denver. All rights + reserved. + + $COPYRIGHT$ + + Additional copyrights may follow + + $HEADER$ + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer listed + in this license in the documentation and/or other materials + provided with the distribution. + + - Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + The copyright holders provide no reassurances that the source code + provided does not infringe any patent, copyright, or any other + intellectual property rights of third parties. The copyright holders + disclaim any liability to any recipient for claims brought against + recipient by any third party for infringement of that parties + intellectual property rights. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + Name: GCC runtime library -Files: .dylibs/* +Files: .libs/libgfortran*.so Description: dynamically linked to files compiled with gcc Availability: https://gcc.gnu.org/viewcvs/gcc/ License: GPLv3 + runtime exception @@ -3401,7 +3462,7 @@ limitations under the License. seldon-core -1.4.0.dev0 +1.5.0.dev0 Apache 2.0 Apache License Version 2.0, January 2004 @@ -4690,7 +4751,7 @@ under the License. tornado -6.0.4 +6.1 Apache Software License Apache License @@ -4952,7 +5013,7 @@ POSSIBILITY OF SUCH DAMAGE. zipp -3.3.1 +3.4.0 MIT License Copyright Jason R. Coombs diff --git a/python/licenses/license_info.csv b/python/licenses/license_info.csv index 12e5494afe..9f5e2ad08c 100644 --- a/python/licenses/license_info.csv +++ b/python/licenses/license_info.csv @@ -4,11 +4,11 @@ "Flask-OpenTracing","1.1.0","BSD License" "Jinja2","2.11.2","BSD License" "Keras-Preprocessing","1.1.2","MIT License" -"Markdown","3.3.2","BSD License" +"Markdown","3.3.3","BSD License" "MarkupSafe","1.1.1","BSD License" "PyYAML","5.3.1","MIT License" "Werkzeug","1.0.1","BSD License" -"absl-py","0.10.0","Apache Software License" +"absl-py","0.11.0","Apache Software License" "astunparse","1.6.3","BSD License" "attrs","20.2.0","MIT License" "cachetools","4.1.1","MIT License" @@ -17,10 +17,10 @@ "click","7.1.2","BSD License" "flatbuffers","1.12","Apache Software License" "gast","0.3.3","BSD License" -"google-auth","1.22.1","Apache Software License" -"google-auth-oauthlib","0.4.1","Apache Software License" +"google-auth","1.23.0","Apache Software License" +"google-auth-oauthlib","0.4.2","Apache Software License" "google-pasta","0.2.0","Apache Software License" -"grpcio","1.32.0","Apache Software License" +"grpcio","1.33.2","Apache Software License" "grpcio-opentracing","1.1.4","Apache Software License" "gunicorn","20.0.4","MIT License" "h5py","2.10.0","BSD License" @@ -29,7 +29,7 @@ "itsdangerous","1.1.0","BSD License" "jaeger-client","4.3.0","Apache Software License" "jsonschema","3.2.0","MIT License" -"numpy","1.19.2","BSD" +"numpy","1.19.4","BSD" "oauthlib","3.1.0","BSD License" "opentracing","2.3.0","Apache Software License" "opt-einsum","3.3.0","MIT" @@ -42,7 +42,7 @@ "requests","2.24.0","Apache Software License" "requests-oauthlib","1.3.0","BSD License" "rsa","4.6","Apache Software License" -"seldon-core","1.4.0.dev0","Apache 2.0" +"seldon-core","1.5.0.dev0","Apache 2.0" "six","1.15.0","MIT License" "tensorboard","2.3.0","Apache Software License" "tensorboard-plugin-wit","1.7.0","Apache 2.0" @@ -51,7 +51,7 @@ "termcolor","1.1.0","MIT License" "threadloop","1.0.2","MIT License" "thrift","0.13.0","Apache Software License" -"tornado","6.0.4","Apache Software License" +"tornado","6.1","Apache Software License" "urllib3","1.25.11","MIT License" "wrapt","1.12.1","BSD License" -"zipp","3.3.1","MIT License" \ No newline at end of file +"zipp","3.4.0","MIT License" \ No newline at end of file diff --git a/python/seldon_core/app.py b/python/seldon_core/app.py index 6be784786e..686d118107 100644 --- a/python/seldon_core/app.py +++ b/python/seldon_core/app.py @@ -5,6 +5,7 @@ from multiprocessing.util import _exit_function from typing import Dict, Union from gunicorn.app.base import BaseApplication +from seldon_core.utils import setup_tracing logger = logging.getLogger(__name__) @@ -68,11 +69,23 @@ class UserModelApplication(StandaloneApplication): user's model. """ - def __init__(self, app, user_object, options: Dict = None): + def __init__( + self, app, user_object, jaeger_extra_tags, interface_name, options: Dict = None + ): self.user_object = user_object + self.jaeger_extra_tags = jaeger_extra_tags + self.interface_name = interface_name super().__init__(app, options) def load(self): + if self.jaeger_extra_tags is not None: + logger.info("Tracing branch is active") + from flask_opentracing import FlaskTracing + + tracer = setup_tracing(self.interface_name) + + logger.info("Set JAEGER_EXTRA_TAGS %s", self.jaeger_extra_tags) + FlaskTracing(tracer, True, self.application, self.jaeger_extra_tags) logger.debug("LOADING APP %d", os.getpid()) try: logger.debug("Calling user load method") diff --git a/python/seldon_core/microservice.py b/python/seldon_core/microservice.py index 3a53554bd3..f28ee9eabb 100644 --- a/python/seldon_core/microservice.py +++ b/python/seldon_core/microservice.py @@ -14,7 +14,7 @@ from seldon_core import persistence, __version__, wrapper as seldon_microservice from seldon_core.metrics import SeldonMetrics from seldon_core.flask_utils import ANNOTATIONS_FILE, SeldonMicroserviceException -from seldon_core.utils import getenv_as_bool +from seldon_core.utils import getenv_as_bool, setup_tracing from seldon_core.app import ( StandaloneApplication, UserModelApplication, @@ -26,7 +26,8 @@ logger = logging.getLogger(__name__) PARAMETERS_ENV_NAME = "PREDICTIVE_UNIT_PARAMETERS" -SERVICE_PORT_ENV_NAME = "PREDICTIVE_UNIT_SERVICE_PORT" +HTTP_SERVICE_PORT_ENV_NAME = "PREDICTIVE_UNIT_HTTP_SERVICE_PORT" +GRPC_SERVICE_PORT_ENV_NAME = "PREDICTIVE_UNIT_GRPC_SERVICE_PORT" METRICS_SERVICE_PORT_ENV_NAME = "PREDICTIVE_UNIT_METRICS_SERVICE_PORT" FILTER_METRICS_ACCESS_LOGS_ENV_NAME = "FILTER_METRICS_ACCESS_LOGS" @@ -34,14 +35,15 @@ LOG_LEVEL_ENV = "SELDON_LOG_LEVEL" DEFAULT_LOG_LEVEL = "INFO" -DEFAULT_PORT = 5000 +DEFAULT_GRPC_PORT = 5000 +DEFAULT_HTTP_PORT = 9000 DEFAULT_METRICS_PORT = 6000 DEBUG_ENV = "SELDON_DEBUG" def start_servers( - target1: Callable, target2: Callable, metrics_target: Callable + target1: Callable, target2: Callable, target3: Callable, metrics_target: Callable ) -> None: """ Start servers @@ -60,10 +62,15 @@ def start_servers( p2.start() p3 = None - if metrics_target: - p3 = mp.Process(target=metrics_target, daemon=True) + if target3: + p3 = mp.Process(target=target3, daemon=True) p3.start() + p4 = None + if metrics_target: + p4 = mp.Process(target=metrics_target, daemon=True) + p4.start() + target1() if p2: @@ -72,6 +79,9 @@ def start_servers( if p3: p3.join() + if p4: + p4.join() + def parse_parameters(parameters: Dict) -> Dict: """ @@ -149,40 +159,6 @@ def load_annotations() -> Dict: return annotations -def setup_tracing(interface_name: str) -> object: - logger.info("Initializing tracing") - from jaeger_client import Config - - jaeger_serv = os.environ.get("JAEGER_AGENT_HOST", "0.0.0.0") - jaeger_port = os.environ.get("JAEGER_AGENT_PORT", 5775) - jaeger_config = os.environ.get("JAEGER_CONFIG_PATH", None) - if jaeger_config is None: - logger.info("Using default tracing config") - config = Config( - config={ # usually read from some yaml config - "sampler": {"type": "const", "param": 1}, - "local_agent": { - "reporting_host": jaeger_serv, - "reporting_port": jaeger_port, - }, - "logging": True, - }, - service_name=interface_name, - validate=True, - ) - else: - logger.info("Loading tracing config from %s", jaeger_config) - import yaml - - with open(jaeger_config, "r") as stream: - config_dict = yaml.load(stream) - config = Config( - config=config_dict, service_name=interface_name, validate=True - ) - # this call also sets opentracing.tracer - return config.initialize_tracer() - - class MetricsEndpointFilter(logging.Filter): def filter(self, record): return seldon_microservice.METRICS_ENDPOINT not in record.getMessage() @@ -228,7 +204,6 @@ def main(): sys.path.append(os.getcwd()) parser = argparse.ArgumentParser() parser.add_argument("interface_name", type=str, help="Name of the user interface.") - parser.add_argument("api_type", type=str, choices=["REST", "GRPC", "FBS"]) parser.add_argument( "--service-type", @@ -298,10 +273,17 @@ def main(): ) parser.add_argument( - "--port", + "--http-port", + type=int, + default=int(os.environ.get(HTTP_SERVICE_PORT_ENV_NAME, DEFAULT_HTTP_PORT)), + help="Set http port of seldon service", + ) + + parser.add_argument( + "--grpc-port", type=int, - default=int(os.environ.get(SERVICE_PORT_ENV_NAME, DEFAULT_PORT)), - help="Set port of seldon service", + default=int(os.environ.get(GRPC_SERVICE_PORT_ENV_NAME, DEFAULT_GRPC_PORT)), + help="Set grpc port of seldon service", ) parser.add_argument( @@ -314,7 +296,7 @@ def main(): ) parser.add_argument( - "--pidfile", type=str, default=None, help="A file path to use for the PID file", + "--pidfile", type=str, default=None, help="A file path to use for the PID file" ) args = parser.parse_args() @@ -351,107 +333,108 @@ def main(): else: user_object = user_class(**parameters) - port = args.port + http_port = args.http_port + grpc_port = args.grpc_port metrics_port = args.metrics_port - if args.tracing: - tracer = setup_tracing(args.interface_name) + # if args.tracing: + # tracer = setup_tracing(args.interface_name) + + seldon_metrics = SeldonMetrics(worker_id_func=os.getpid) + # TODO why 2 ways to create metrics server + # seldon_metrics = SeldonMetrics( + # worker_id_func=lambda: threading.current_thread().name + # ) + if args.debug: + # Start Flask debug server + def rest_prediction_server(): + app = seldon_microservice.get_rest_microservice(user_object, seldon_metrics) + try: + user_object.load() + except (NotImplementedError, AttributeError): + pass + if args.tracing: + logger.info("Tracing branch is active") + from flask_opentracing import FlaskTracing - if args.api_type == "REST": - seldon_metrics = SeldonMetrics(worker_id_func=os.getpid) + tracer = setup_tracing(args.interface_name) - if args.debug: - # Start Flask debug server - def rest_prediction_server(): - app = seldon_microservice.get_rest_microservice( - user_object, seldon_metrics - ) - try: - user_object.load() - except (NotImplementedError, AttributeError): - pass - if args.tracing: - logger.info("Tracing branch is active") - from flask_opentracing import FlaskTracing - - logger.info("Set JAEGER_EXTRA_TAGS %s", jaeger_extra_tags) - FlaskTracing(tracer, True, app, jaeger_extra_tags) - - app.run( - host="0.0.0.0", - port=port, - threaded=False if args.single_threaded else True, - ) + logger.info("Set JAEGER_EXTRA_TAGS %s", jaeger_extra_tags) + FlaskTracing(tracer, True, app, jaeger_extra_tags) - logger.info( - "REST microservice running on port %i single-threaded=%s", - port, - args.single_threaded, + app.run( + host="0.0.0.0", + port=http_port, + threaded=False if args.single_threaded else True, ) - server1_func = rest_prediction_server - else: - # Start production server - def rest_prediction_server(): - options = { - "bind": "%s:%s" % ("0.0.0.0", port), - "accesslog": accesslog(args.log_level), - "loglevel": args.log_level.lower(), - "timeout": 5000, - "threads": threads(args.threads, args.single_threaded), - "workers": args.workers, - "max_requests": args.max_requests, - "max_requests_jitter": args.max_requests_jitter, - "post_worker_init": post_worker_init, - } - if args.pidfile is not None: - options["pidfile"] = args.pidfile - app = seldon_microservice.get_rest_microservice( - user_object, seldon_metrics - ) - UserModelApplication(app, user_object, options=options).run() - logger.info("REST gunicorn microservice running on port %i", port) - server1_func = rest_prediction_server - - elif args.api_type == "GRPC": - seldon_metrics = SeldonMetrics( - worker_id_func=lambda: threading.current_thread().name + logger.info( + "REST microservice running on port %i single-threaded=%s", + http_port, + args.single_threaded, ) + server1_func = rest_prediction_server + else: + # Start production server + def rest_prediction_server(): + options = { + "bind": "%s:%s" % ("0.0.0.0", http_port), + "accesslog": accesslog(args.log_level), + "loglevel": args.log_level.lower(), + "timeout": 5000, + "threads": threads(args.threads, args.single_threaded), + "workers": args.workers, + "max_requests": args.max_requests, + "max_requests_jitter": args.max_requests_jitter, + "post_worker_init": post_worker_init, + } + if args.pidfile is not None: + options["pidfile"] = args.pidfile + app = seldon_microservice.get_rest_microservice(user_object, seldon_metrics) - def grpc_prediction_server(): + UserModelApplication( + app, + user_object, + jaeger_extra_tags, + args.interface_name, + options=options, + ).run() - if args.tracing: - from grpc_opentracing import open_tracing_server_interceptor + logger.info("REST gunicorn microservice running on port %i", http_port) + server1_func = rest_prediction_server - logger.info("Adding tracer") - interceptor = open_tracing_server_interceptor(tracer) - else: - interceptor = None + def grpc_prediction_server(): - server = seldon_microservice.get_grpc_server( - user_object, - seldon_metrics, - annotations=annotations, - trace_interceptor=interceptor, - ) + if args.tracing: + from grpc_opentracing import open_tracing_server_interceptor - try: - user_object.load() - except (NotImplementedError, AttributeError): - pass + logger.info("Adding tracer") + tracer = setup_tracing(args.interface_name) + interceptor = open_tracing_server_interceptor(tracer) + else: + interceptor = None - server.add_insecure_port(f"0.0.0.0:{port}") + server = seldon_microservice.get_grpc_server( + user_object, + seldon_metrics, + annotations=annotations, + trace_interceptor=interceptor, + ) - server.start() + try: + user_object.load() + except (NotImplementedError, AttributeError): + pass - logger.info("GRPC microservice Running on port %i", port) - while True: - time.sleep(1000) + server.add_insecure_port(f"0.0.0.0:{grpc_port}") - server1_func = grpc_prediction_server + server.start() - else: - server1_func = None + logger.info("GRPC microservice Running on port %i", grpc_port) + while True: + time.sleep(1000) + + server2_func = grpc_prediction_server def rest_metrics_server(): app = seldon_microservice.get_metrics_microservice(seldon_metrics) @@ -477,12 +460,12 @@ def rest_metrics_server(): if hasattr(user_object, "custom_service") and callable( getattr(user_object, "custom_service") ): - server2_func = user_object.custom_service + server3_func = user_object.custom_service else: - server2_func = None + server3_func = None logger.info("Starting servers") - start_servers(server1_func, server2_func, metrics_server_func) + start_servers(server1_func, server2_func, server3_func, metrics_server_func) if __name__ == "__main__": diff --git a/python/seldon_core/utils.py b/python/seldon_core/utils.py index 9978e183be..fc7511e994 100644 --- a/python/seldon_core/utils.py +++ b/python/seldon_core/utils.py @@ -3,6 +3,7 @@ import sys import base64 import numpy as np +import logging from google.protobuf import json_format, any_pb2 from google.protobuf.json_format import MessageToDict, ParseDict @@ -23,6 +24,8 @@ import tensorflow as tf from tensorflow.core.framework.tensor_pb2 import TensorProto +logger = logging.getLogger(__name__) + ENV_MODEL_NAME = "PREDICTIVE_UNIT_ID" ENV_MODEL_IMAGE = "PREDICTIVE_UNIT_IMAGE" @@ -715,3 +718,37 @@ def getenv_as_bool(*env_vars, default=False): return default return val.lower() in ["1", "true", "t"] + + +def setup_tracing(interface_name: str) -> object: + logger.info("Initializing tracing") + from jaeger_client import Config + + jaeger_serv = os.environ.get("JAEGER_AGENT_HOST", "0.0.0.0") + jaeger_port = os.environ.get("JAEGER_AGENT_PORT", 5775) + jaeger_config = os.environ.get("JAEGER_CONFIG_PATH", None) + if jaeger_config is None: + logger.info("Using default tracing config") + config = Config( + config={ # usually read from some yaml config + "sampler": {"type": "const", "param": 1}, + "local_agent": { + "reporting_host": jaeger_serv, + "reporting_port": jaeger_port, + }, + "logging": True, + }, + service_name=interface_name, + validate=True, + ) + else: + logger.info("Loading tracing config from %s", jaeger_config) + import yaml + + with open(jaeger_config, "r") as stream: + config_dict = yaml.load(stream) + config = Config( + config=config_dict, service_name=interface_name, validate=True + ) + # this call also sets opentracing.tracer + return config.initialize_tracer() diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 20cedb4098..06825058c3 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -35,12 +35,9 @@ def microservice(request): app_name = opts.get("app_name", "model-template-app") app_location = opts.get("app_location", os.path.join(RESOURCES_PATH, app_name)) envs = opts.get("envs", {}) - grpc = opts.get("grpc", False) tracing = opts.get("tracing", False) - wrapper = MicroserviceWrapper( - app_location=app_location, envs=envs, grpc=grpc, tracing=tracing - ) + wrapper = MicroserviceWrapper(app_location=app_location, envs=envs, tracing=tracing) with wrapper: yield wrapper diff --git a/python/tests/helpers.py b/python/tests/helpers.py index c71ec68dbd..ec38b16aa9 100644 --- a/python/tests/helpers.py +++ b/python/tests/helpers.py @@ -16,12 +16,12 @@ def predict(self, X, features_names): class MicroserviceWrapper: - def __init__(self, app_location, envs={}, grpc=False, tracing=False): + def __init__(self, app_location, envs={}, tracing=False): self.app_location = app_location - self.env_vars = self._env_vars(envs, grpc) + self.env_vars = self._env_vars(envs) self.cmd = self._get_cmd(tracing) - def _env_vars(self, envs, grpc): + def _env_vars(self, envs): env_vars = dict(os.environ) env_vars.update(envs) env_vars.update( @@ -29,7 +29,8 @@ def _env_vars(self, envs, grpc): "PYTHONUNBUFFERED": "x", "PYTHONPATH": self.app_location, "APP_HOST": "127.0.0.1", - "PREDICTIVE_UNIT_SERVICE_PORT": "5000", + "PREDICTIVE_UNIT_HTTP_SERVICE_PORT": "9000", + "PREDICTIVE_UNIT_GRPC_SERVICE_PORT": "5000", "PREDICTIVE_UNIT_METRICS_SERVICE_PORT": "6005", "PREDICTIVE_UNIT_METRICS_ENDPOINT": "/metrics-endpoint", } @@ -45,16 +46,12 @@ def _env_vars(self, envs, grpc): if key and value: env_vars[key] = value - if grpc: - env_vars["API_TYPE"] = "GRPC" - return env_vars def _get_cmd(self, tracing): cmd = ( "seldon-core-microservice", self.env_vars["MODEL_NAME"], - self.env_vars["API_TYPE"], "--service-type", self.env_vars["SERVICE_TYPE"], "--persistence", @@ -85,10 +82,13 @@ def __enter__(self): def _wait_until_ready(self): logging.debug("=== trying again") s1 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - r1 = s1.connect_ex(("127.0.0.1", 5000)) + r1 = s1.connect_ex(("127.0.0.1", 9000)) s2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) r2 = s2.connect_ex(("127.0.0.1", 6005)) - if r1 != 0 or r2 != 0: + s3 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + r3 = s3.connect_ex(("127.0.0.1", 5000)) + + if r1 != 0 or r2 != 0 or r3 != 0: raise EOFError("Server not ready yet") logging.info("microservice ready") diff --git a/python/tests/test_microservice.py b/python/tests/test_microservice.py index bf3e0b9d67..e530667a8b 100644 --- a/python/tests/test_microservice.py +++ b/python/tests/test_microservice.py @@ -37,7 +37,7 @@ def test_microservice_version(): def test_model_template_app_rest(microservice): data = '{"data":{"names":["a","b"],"ndarray":[[1.0,2.0]]}}' - response = requests.get("http://127.0.0.1:5000/predict", params="json=%s" % data) + response = requests.get("http://127.0.0.1:9000/predict", params="json=%s" % data) response.raise_for_status() assert response.json() == { "data": {"names": ["t:0", "t:1"], "ndarray": [[1.0, 2.0]]}, @@ -50,7 +50,7 @@ def test_model_template_app_rest(microservice): '"ndarray":[[1.0,2.0]]}},"reward":1}' ) response = requests.get( - "http://127.0.0.1:5000/send-feedback", params="json=%s" % data + "http://127.0.0.1:9000/send-feedback", params="json=%s" % data ) response.raise_for_status() assert response.json() == {"data": {"ndarray": []}, "meta": {}} @@ -58,7 +58,7 @@ def test_model_template_app_rest(microservice): def test_model_template_app_rest_tags(microservice): data = '{"meta":{"tags":{"foo":"bar"}},"data":{"names":["a","b"],"ndarray":[[1.0,2.0]]}}' - response = requests.get("http://127.0.0.1:5000/predict", params="json=%s" % data) + response = requests.get("http://127.0.0.1:9000/predict", params="json=%s" % data) response.raise_for_status() assert response.json() == { "data": {"names": ["t:0", "t:1"], "ndarray": [[1.0, 2.0]]}, @@ -68,7 +68,7 @@ def test_model_template_app_rest_tags(microservice): def test_model_template_app_rest_metrics(microservice): data = '{"meta":{"metrics":[{"key":"mygauge","type":"GAUGE","value":100}]},"data":{"names":["a","b"],"ndarray":[[1.0,2.0]]}}' - response = requests.get("http://127.0.0.1:5000/predict", params="json=%s" % data) + response = requests.get("http://127.0.0.1:9000/predict", params="json=%s" % data) response.raise_for_status() assert response.json() == { "data": {"names": ["t:0", "t:1"], "ndarray": [[1.0, 2.0]]}, @@ -87,7 +87,7 @@ def test_model_template_app_rest_metrics_endpoint(microservice): ) def test_model_template_app_rest_submodule(microservice): data = '{"data":{"names":["a","b"],"ndarray":[[1.0,2.0]]}}' - response = requests.get("http://127.0.0.1:5000/predict", params="json=%s" % data) + response = requests.get("http://127.0.0.1:9000/predict", params="json=%s" % data) response.raise_for_status() assert response.json() == { "data": {"names": ["t:0", "t:1"], "ndarray": [[1.0, 2.0]]}, @@ -100,13 +100,12 @@ def test_model_template_app_rest_submodule(microservice): '"ndarray":[[1.0,2.0]]}},"reward":1}' ) response = requests.get( - "http://127.0.0.1:5000/send-feedback", params="json=%s" % data + "http://127.0.0.1:9000/send-feedback", params="json=%s" % data ) response.raise_for_status() assert response.json() == {"data": {"ndarray": []}, "meta": {}} -@pytest.mark.parametrize("microservice", [{"grpc": True}], indirect=True) def test_model_template_app_grpc(microservice): data = np.array([[1, 2]]) datadef = prediction_pb2.DefaultData( @@ -130,7 +129,6 @@ def test_model_template_app_grpc(microservice): response = stub.SendFeedback(request=request) -@pytest.mark.parametrize("microservice", [{"grpc": True}], indirect=True) def test_model_template_app_grpc_tags(microservice): data = np.array([[1, 2]]) datadef = prediction_pb2.DefaultData( @@ -152,7 +150,6 @@ def test_model_template_app_grpc_tags(microservice): assert response.meta.tags["foo"].string_value == "bar" -@pytest.mark.parametrize("microservice", [{"grpc": True}], indirect=True) def test_model_template_app_grpc_metrics(microservice): data = np.array([[1, 2]]) datadef = prediction_pb2.DefaultData( @@ -193,7 +190,7 @@ def test_model_template_app_grpc_metrics(microservice): ) def test_model_template_app_tracing_config(microservice): data = '{"data":{"names":["a","b"],"ndarray":[[1.0,2.0]]}}' - response = requests.get("http://127.0.0.1:5000/predict", params="json=%s" % data) + response = requests.get("http://127.0.0.1:9000/predict", params="json=%s" % data) response.raise_for_status() assert response.json() == { "data": {"names": ["t:0", "t:1"], "ndarray": [[1.0, 2.0]]}, @@ -206,7 +203,7 @@ def test_model_template_app_tracing_config(microservice): '"ndarray":[[1.0,2.0]]}},"reward":1}' ) response = requests.get( - "http://127.0.0.1:5000/send-feedback", params="json=%s" % data + "http://127.0.0.1:9000/send-feedback", params="json=%s" % data ) response.raise_for_status() assert response.json() == {"data": {"ndarray": []}, "meta": {}} @@ -216,7 +213,6 @@ def test_model_template_bad_params(): params = [ join(dirname(__file__), "model-template-app"), "seldon-core-microservice", - "REST", "--parameters", '[{"type":"FLOAT","name":"foo","value":"abc"}]', ] @@ -229,7 +225,6 @@ def test_model_template_bad_params_type(): params = [ join(dirname(__file__), "model-template-app"), "seldon-core-microservice", - "REST", "--parameters", '[{"type":"FOO","name":"foo","value":"abc"}]', ] diff --git a/release.py b/release.py index 09c653ac54..bf6c226512 100644 --- a/release.py +++ b/release.py @@ -77,8 +77,7 @@ def update_pom_file(fpath, seldon_core_version, debug=False): ] err, out = run_command(args, debug) - ##pp(out) - ##pp(err) + if err == None: print("updated {fpath}".format(**locals())) else: @@ -104,6 +103,26 @@ def update_chart_yaml_file(fpath, seldon_core_version, debug=False): print("updated {fpath}".format(**locals())) +def update_helm_values_yaml_file_default_images(fpath, seldon_core_version, debug=False): + fpath = os.path.realpath(fpath) + if debug: + print("processing [{}]".format(fpath)) + args = [ + "sed", + "-i", + "s/version: \(.*\)/version: {seldon_core_version}/".format( + **locals() + ), + fpath, + ] + err, out = run_command(args, debug) + + if err == None: + print("updated helm values yaml for default images".format(**locals())) + else: + print("error updating helm values yaml for default images".format(**locals())) + print(err) + def update_operator_values_yaml_file_core_images(fpath, seldon_core_version, debug=False): fpath = os.path.realpath(fpath) if debug: @@ -117,8 +136,7 @@ def update_operator_values_yaml_file_core_images(fpath, seldon_core_version, deb fpath, ] err, out = run_command(args, debug) - # pp(out) - # pp(err) + if err == None: print("updated operator values yaml for core images".format(**locals())) else: @@ -139,8 +157,7 @@ def update_operator_values_yaml_file_prepackaged_images(current_seldon_core_vers fpath, ] err, out = run_command(args, debug) - # pp(out) - # pp(err) + if err == None: print("updated operator values yaml for prepackaged server images".format(**locals())) else: @@ -160,8 +177,7 @@ def update_operator_values_yaml_file_explainer_image(fpath, seldon_core_version, fpath, ] err, out = run_command(args, debug) - # pp(out) - # pp(err) + if err == None: print("updated operator values yaml for prepackaged server images".format(**locals())) else: @@ -183,8 +199,7 @@ def update_operator_kustomize_prepackaged_images(current_seldon_core_version, fp fpath, ] err, out = run_command(args, debug) - # pp(out) - # pp(err) + if err == None: print("updated operator kustomize yaml for prepackaged server images".format(**locals())) else: @@ -230,8 +245,7 @@ def update_kustomize_engine_version(seldon_core_version, debug=False): "operator/config/manager/manager.yaml", ] err, out = run_command(args, debug) - # pp(out) - # pp(err) + if err == None: print("updated kustomize".format(**locals())) else: @@ -248,8 +262,7 @@ def update_kustomize_executor_version(seldon_core_version, debug=False): "operator/config/manager/manager.yaml", ] err, out = run_command(args, debug) - # pp(out) - # pp(err) + if err == None: print("updated kustomize".format(**locals())) else: @@ -277,7 +290,7 @@ def update_operator_version(seldon_core_version, debug=False): def update_image_metadata_json(seldon_core_version, debug=False): paths = [ "examples/models/mean_classifier/image_metadata.json", - "integrations/tfserving/image_metadata.json", + "servers/tfserving_proxy/image_metadata.json", "servers/sklearnserver/sklearnserver/image_metadata.json", "servers/mlflowserver/mlflowserver/image_metadata.json", "servers/xgboostserver/xgboostserver/image_metadata.json" @@ -329,8 +342,7 @@ def update_python_wrapper_fixed_versions(seldon_core_version, debug=False): "{seldon_core_version}".format(**locals()), ] err, out = run_command(args, debug) - # pp(out) - # pp(err) + if err == None: print("Updated python wrapper in matching files".format(**locals())) else: @@ -344,7 +356,9 @@ def set_version( chart_yaml_files, operator_values_yaml_file, operator_kustomize_yaml_file, - debug=False, + abtest_yaml_file, + mab_yaml_file, + debug=False ): update_python_wrapper_fixed_versions(seldon_core_version, debug) @@ -361,6 +375,16 @@ def set_version( if operator_kustomize_yaml_file != None else None ) + abtest_yaml_file_realpath = ( + os.path.realpath(abtest_yaml_file) + if abtest_yaml_file != None + else None + ) + mab_values_yaml_file_realpath = ( + os.path.realpath(mab_yaml_file) + if mab_yaml_file != None + else None + ) # Update kustomize update_kustomize_engine_version(seldon_core_version, debug) @@ -389,6 +413,16 @@ def set_version( operator_values_yaml_file_realpath, seldon_core_version, debug ) + # update the operator helm values files + if mab_yaml_file != None: + update_helm_values_yaml_file_default_images( + mab_values_yaml_file_realpath, seldon_core_version, debug + ) + if abtest_yaml_file != None: + update_helm_values_yaml_file_default_images( + abtest_yaml_file_realpath, seldon_core_version, debug + ) + if operator_values_yaml_file != None: update_operator_values_yaml_file_prepackaged_images( current_seldon_core_version, operator_values_yaml_file_realpath, seldon_core_version, debug @@ -417,6 +451,8 @@ def main(argv): ] OPERATOR_VALUES_YAML_FILE = "helm-charts/seldon-core-operator/values.yaml" OPERATOR_KUSTOMIZE_CONFIGMAP = "operator/config/manager/configmap.yaml" + AB_VALUES_YAML_FILE = "helm-charts/seldon-abtest/values.yaml" + MAB_VALUES_YAML_FILE = "helm-charts/seldon-mab/values.yaml" opts = getOpts(argv[1:]) current_version = get_current_version() @@ -429,7 +465,9 @@ def main(argv): CHART_YAML_FILES, OPERATOR_VALUES_YAML_FILE, OPERATOR_KUSTOMIZE_CONFIGMAP, - opts.debug, + AB_VALUES_YAML_FILE, + MAB_VALUES_YAML_FILE, + opts.debug ) print("done") diff --git a/servers/mlflowserver/Makefile b/servers/mlflowserver/Makefile index 92a8d3e640..accba1e910 100644 --- a/servers/mlflowserver/Makefile +++ b/servers/mlflowserver/Makefile @@ -1,43 +1,32 @@ SHELL := /bin/bash VERSION := $(shell cat ../../version.txt) IMAGE_NAME_BASE=mlflowserver -IMAGE_BASE=seldonio/${IMAGE_NAME_BASE} +IMAGE_NAME=seldonio/${IMAGE_NAME_BASE} KIND_NAME ?= kind -build_%: +build: s2i build \ - -E environment_$* \ + -E environment \ ./mlflowserver \ seldonio/seldon-core-s2i-python37-ubi8:${VERSION} \ - ${IMAGE_BASE}_$*:${VERSION} + ${IMAGE_NAME}:${VERSION} -push_%: - docker push ${IMAGE_BASE}_$*:${VERSION} - -kind_load_%: - kind load -v 3 docker-image ${IMAGE_BASE}_$*:${VERSION} --name ${KIND_NAME} - -.PHONY: push_all -push_all: push_rest push_grpc - -.PHONY: build_all -build_all: build_rest build_grpc - -.PHONY: kind_load_all -kind_load: build_all kind_load_rest kind_load_grpc +push: + docker push ${IMAGE_NAME}_$*:${VERSION} +kind_load: build + kind load -v 3 docker-image ${IMAGE_NAME}:${VERSION} --name ${KIND_NAME} # https://connect.redhat.com/project/4121681/view scan_rest=ospid-fe6af4a5-e0c6-414a-b0a6-00b7773b6336 # https://connect.redhat.com/project/4122001/view scan_grpc=ospid-fbd924dd-bc1e-403b-9f35-5f75b82b1cdf -redhat-image-scan-%: - docker pull ${IMAGE_BASE}_$*:${VERSION} +# +scan= +redhat-image-scan: + docker pull ${IMAGE_NAME}:${VERSION} source ~/.config/seldon/seldon-core/redhat-image-passwords.sh && \ - echo $${rh_password_mlflowserver_$*} | docker login -u unused scan.connect.redhat.com --password-stdin - docker tag ${IMAGE_BASE}_$*:${VERSION} scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} - docker push scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} - -.PHONY: redhat-image-scan -redhat-image-scan: redhat-image-scan-rest redhat-image-scan-grpc + echo $${rh_password_mlflowserver} | docker login -u unused scan.connect.redhat.com --password-stdin + docker tag ${IMAGE_BASE}:${VERSION} scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} + docker push scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} diff --git a/servers/mlflowserver/environment_grpc b/servers/mlflowserver/environment similarity index 84% rename from servers/mlflowserver/environment_grpc rename to servers/mlflowserver/environment index 189e2c54f3..73be9d8071 100644 --- a/servers/mlflowserver/environment_grpc +++ b/servers/mlflowserver/environment @@ -1,5 +1,4 @@ MODEL_NAME=MLFlowServer -API_TYPE=GRPC SERVICE_TYPE=MODEL PERSISTENCE=0 CONDA_ENV_NAME=mlflow diff --git a/servers/mlflowserver/environment_rest b/servers/mlflowserver/environment_rest deleted file mode 100644 index bb58b80a19..0000000000 --- a/servers/mlflowserver/environment_rest +++ /dev/null @@ -1,5 +0,0 @@ -MODEL_NAME=MLFlowServer -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 -CONDA_ENV_NAME=mlflow diff --git a/servers/mlflowserver/samples/elasticnet_wine.yaml b/servers/mlflowserver/samples/elasticnet_wine.yaml index 830f32957c..ddc78b7509 100644 --- a/servers/mlflowserver/samples/elasticnet_wine.yaml +++ b/servers/mlflowserver/samples/elasticnet_wine.yaml @@ -12,8 +12,8 @@ spec: containers: - name: classifier livenessProbe: - initialDelaySeconds: 60 - failureThreshold: 100 + initialDelaySeconds: 80 + failureThreshold: 200 periodSeconds: 5 successThreshold: 1 httpGet: @@ -21,8 +21,8 @@ spec: port: http scheme: HTTP readinessProbe: - initialDelaySeconds: 60 - failureThreshold: 100 + initialDelaySeconds: 80 + failureThreshold: 200 periodSeconds: 5 successThreshold: 1 httpGet: @@ -35,4 +35,4 @@ spec: modelUri: gs://seldon-models/mlflow/elasticnet_wine name: classifier name: default - replicas: 1 + replicas: 1 \ No newline at end of file diff --git a/servers/sklearnserver/Makefile b/servers/sklearnserver/Makefile index 7a8e4f264b..f5b66ae439 100644 --- a/servers/sklearnserver/Makefile +++ b/servers/sklearnserver/Makefile @@ -1,43 +1,34 @@ SHELL := /bin/bash VERSION := $(shell cat ../../version.txt) IMAGE_NAME_BASE=sklearnserver -IMAGE_BASE=seldonio/${IMAGE_NAME_BASE} +IMAGE_NAME=seldonio/${IMAGE_NAME_BASE} KIND_NAME ?= kind -build_%: +build: s2i build \ - -E environment_$* \ + -E environment \ ./sklearnserver \ seldonio/seldon-core-s2i-python37-ubi8:${VERSION} \ - ${IMAGE_BASE}_$*:${VERSION} + ${IMAGE_NAME}:${VERSION} -push_%: - docker push ${IMAGE_BASE}_$*:${VERSION} - -kind_load_%: - kind load -v 3 docker-image ${IMAGE_BASE}_$*:${VERSION} --name ${KIND_NAME} - -.PHONY: push_all -push_all: push_rest push_grpc - -.PHONY: build_all -build_all: build_rest build_grpc - -.PHONY: kind_load_all -kind_load: build_all kind_load_rest kind_load_grpc +push: + docker push ${IMAGE_NAME}:${VERSION} +kind_load: build + kind load -v 3 docker-image ${IMAGE_NAME}:${VERSION} --name ${KIND_NAME} # https://connect.redhat.com/project/4060551/view scan_rest=ospid-5c2f945c-69fa-4de4-8574-e9a61f69d69a # https://connect.redhat.com/project/4061631/view scan_grpc=ospid-d75d8767-c689-4368-9859-4ec4f0062a3a -redhat-image-scan-%: - docker pull ${IMAGE_BASE}_$*:${VERSION} + +# NEED NEW PROJECT +scan= +redhat-image-scan: + docker pull ${IMAGE_NAME}:${VERSION} source ~/.config/seldon/seldon-core/redhat-image-passwords.sh && \ - echo $${rh_password_sklearnserver_$*} | docker login -u unused scan.connect.redhat.com --password-stdin - docker tag ${IMAGE_BASE}_$*:${VERSION} scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} - docker push scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} + echo $${rh_password_sklearnserver} | docker login -u unused scan.connect.redhat.com --password-stdin + docker tag ${IMAGE_NAME}:${VERSION} scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} + docker push scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} -.PHONY: redhat-image-scan -redhat-image-scan: redhat-image-scan-rest redhat-image-scan-grpc diff --git a/servers/sklearnserver/environment_grpc b/servers/sklearnserver/environment similarity index 80% rename from servers/sklearnserver/environment_grpc rename to servers/sklearnserver/environment index 7bf468f3ec..8bf8f4a9db 100644 --- a/servers/sklearnserver/environment_grpc +++ b/servers/sklearnserver/environment @@ -1,4 +1,3 @@ MODEL_NAME=SKLearnServer -API_TYPE=GRPC SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/servers/sklearnserver/environment_rest b/servers/sklearnserver/environment_rest deleted file mode 100644 index 55e877a05c..0000000000 --- a/servers/sklearnserver/environment_rest +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=SKLearnServer -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/servers/sklearnserver/samples/iris_grpc.yaml b/servers/sklearnserver/samples/iris_grpc.yaml deleted file mode 100644 index f213bbb84c..0000000000 --- a/servers/sklearnserver/samples/iris_grpc.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: machinelearning.seldon.io/v1alpha2 -kind: SeldonDeployment -metadata: - name: sklearn -spec: - name: iris - transport: grpc - predictors: - - graph: - children: [] - implementation: SKLEARN_SERVER - modelUri: gs://seldon-models/sklearn/iris - name: classifier - name: default - replicas: 1 - svcOrchSpec: - env: - - name: SELDON_LOG_LEVEL - value: DEBUG diff --git a/servers/tfserving/samples/halfplustwo_rest.yaml b/servers/tfserving/samples/halfplustwo_rest.yaml index d9c3f1932d..f3e5c4d032 100644 --- a/servers/tfserving/samples/halfplustwo_rest.yaml +++ b/servers/tfserving/samples/halfplustwo_rest.yaml @@ -17,5 +17,4 @@ spec: type: STRING value: halfplustwo name: default - replicas: 1 - + replicas: 1 \ No newline at end of file diff --git a/servers/tfserving/samples/mnist_rest.yaml b/servers/tfserving/samples/mnist_rest.yaml index 3ff2897eb0..ebdfcb2e80 100644 --- a/servers/tfserving/samples/mnist_rest.yaml +++ b/servers/tfserving/samples/mnist_rest.yaml @@ -17,5 +17,11 @@ spec: - name: model_name type: STRING value: mnist-model + - name: model_input + type: STRING + value: images + - name: model_output + type: STRING + value: scores name: default - replicas: 1 + replicas: 1 \ No newline at end of file diff --git a/integrations/tfserving/.s2i/bin/assemble b/servers/tfserving_proxy/.s2i/bin/assemble similarity index 100% rename from integrations/tfserving/.s2i/bin/assemble rename to servers/tfserving_proxy/.s2i/bin/assemble diff --git a/servers/tfserving_proxy/Makefile b/servers/tfserving_proxy/Makefile new file mode 100644 index 0000000000..b74070c870 --- /dev/null +++ b/servers/tfserving_proxy/Makefile @@ -0,0 +1,32 @@ +SHELL := /bin/bash +VERSION := $(shell cat ../../version.txt) +IMAGE_NAME_BASE=tfserving-proxy +IMAGE_NAME=seldonio/${IMAGE_NAME_BASE} +KIND_NAME ?= kind + +build: + s2i build \ + -E environment \ + . \ + seldonio/seldon-core-s2i-python37-ubi8:${VERSION} \ + ${IMAGE_NAME}:${VERSION} + +push: + docker push ${IMAGE_NAME}:${VERSION} + +kind_load: build + kind load -v 3 docker-image ${IMAGE_NAME}:${VERSION} --name ${KIND_NAME} + +# https://connect.redhat.com/project/4098981/view +scan_rest=ospid-2cbfde5f-10d2-4cc1-9b1d-0dff7f6d1021 +# https://connect.redhat.com/project/4100321/view +scan_grpc=ospid-da09f299-64d1-4e19-86c9-af60a0ddb851 +# +scan= +redhat-image-scan: + docker pull ${IMAGE_NAME}:${VERSION} + source ~/.config/seldon/seldon-core/redhat-image-passwords.sh && \ + echo $${rh_password_tfproxy} | docker login -u unused scan.connect.redhat.com --password-stdin + docker tag ${IMAGE_NAME}:${VERSION} scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}_$*:${VERSION} + docker push scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} + diff --git a/integrations/tfserving/README.md b/servers/tfserving_proxy/README.md similarity index 100% rename from integrations/tfserving/README.md rename to servers/tfserving_proxy/README.md diff --git a/integrations/tfserving/TfServingProxy.py b/servers/tfserving_proxy/TfServingProxy.py similarity index 87% rename from integrations/tfserving/TfServingProxy.py rename to servers/tfserving_proxy/TfServingProxy.py index 7d38beacdd..bccfc98ae1 100644 --- a/integrations/tfserving/TfServingProxy.py +++ b/servers/tfserving_proxy/TfServingProxy.py @@ -32,17 +32,17 @@ def __init__( model_output=None): log.debug("rest_endpoint:",rest_endpoint) log.debug("grpc_endpoint:",grpc_endpoint) - if not grpc_endpoint is None: - self.grpc = True - max_msg = 1000000000 - options = [('grpc.max_message_length', max_msg), - ('grpc.max_send_message_length', max_msg), - ('grpc.max_receive_message_length', max_msg)] - channel = grpc.insecure_channel(grpc_endpoint,options) - self.stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) - else: - self.grpc = False - self.rest_endpoint = rest_endpoint+"/v1/models/"+model_name+":predict" + + # grpc + max_msg = 1000000000 + options = [('grpc.max_message_length', max_msg), + ('grpc.max_send_message_length', max_msg), + ('grpc.max_receive_message_length', max_msg)] + channel = grpc.insecure_channel(grpc_endpoint,options) + self.stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + + # rest + self.rest_endpoint = rest_endpoint+"/v1/models/"+model_name+":predict" self.model_name = model_name if signature_name is None: self.signature_name = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY @@ -80,7 +80,7 @@ def predict_grpc(self,request): else: data_arr = grpc_datadef_to_array(request.data) tfrequest.inputs[self.model_input].CopyFrom( - tf.contrib.util.make_tensor_proto( + tf.make_tensor_proto( data_arr.tolist(), shape=data_arr.shape)) result = self.stub.Predict(tfrequest) diff --git a/examples/models/sk_mnist/environment_rest b/servers/tfserving_proxy/environment similarity index 50% rename from examples/models/sk_mnist/environment_rest rename to servers/tfserving_proxy/environment index 840dc086f5..d89c83830e 100644 --- a/examples/models/sk_mnist/environment_rest +++ b/servers/tfserving_proxy/environment @@ -1,4 +1,3 @@ -MODEL_NAME=SkMnist -API_TYPE=REST +MODEL_NAME=TfServingProxy SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/integrations/tfserving/environment_grpc b/servers/tfserving_proxy/environment_grpc similarity index 100% rename from integrations/tfserving/environment_grpc rename to servers/tfserving_proxy/environment_grpc diff --git a/integrations/tfserving/environment_rest b/servers/tfserving_proxy/environment_rest similarity index 100% rename from integrations/tfserving/environment_rest rename to servers/tfserving_proxy/environment_rest diff --git a/integrations/tfserving/image_metadata.json b/servers/tfserving_proxy/image_metadata.json similarity index 100% rename from integrations/tfserving/image_metadata.json rename to servers/tfserving_proxy/image_metadata.json diff --git a/integrations/tfserving/requirements.txt b/servers/tfserving_proxy/requirements.txt similarity index 100% rename from integrations/tfserving/requirements.txt rename to servers/tfserving_proxy/requirements.txt diff --git a/integrations/tfserving/test_tfserving_proxy b/servers/tfserving_proxy/test_tfserving_proxy similarity index 100% rename from integrations/tfserving/test_tfserving_proxy rename to servers/tfserving_proxy/test_tfserving_proxy diff --git a/integrations/tfserving/test_tfserving_proxy.py b/servers/tfserving_proxy/test_tfserving_proxy.py similarity index 100% rename from integrations/tfserving/test_tfserving_proxy.py rename to servers/tfserving_proxy/test_tfserving_proxy.py diff --git a/servers/xgboostserver/Makefile b/servers/xgboostserver/Makefile index 6421f7399a..525c90838e 100644 --- a/servers/xgboostserver/Makefile +++ b/servers/xgboostserver/Makefile @@ -1,42 +1,33 @@ SHELL := /bin/bash VERSION := $(shell cat ../../version.txt) IMAGE_NAME_BASE=xgboostserver -IMAGE_BASE=seldonio/${IMAGE_NAME_BASE} +IMAGE_NAME=seldonio/${IMAGE_NAME_BASE} KIND_NAME ?= kind -build_%: +build: s2i build \ - -E environment_$* \ + -E environment \ ./xgboostserver \ seldonio/seldon-core-s2i-python37-ubi8:${VERSION} \ - ${IMAGE_BASE}_$*:${VERSION} + ${IMAGE_NAME}:${VERSION} -push_%: - docker push ${IMAGE_BASE}_$*:${VERSION} +push: + docker push ${IMAGE_NAME}:${VERSION} -kind_load_%: - kind load -v 3 docker-image ${IMAGE_BASE}_$*:${VERSION} --name ${KIND_NAME} - -.PHONY: push_all -push_all: push_rest push_grpc - -.PHONY: build_all -build_all: build_rest build_grpc - -.PHONY: kind_load_all -kind_load: build_all kind_load_rest kind_load_grpc +kind_load: build + kind load -v 3 docker-image ${IMAGE_NAME}:${VERSION} --name ${KIND_NAME} # https://connect.redhat.com/project/4127491/view scan_rest=ospid-096b89ca-b4a6-4cff-8c13-d65945a5adb9 # scan_grpc=ospid-ba6387aa-1144-4764-8b04-80e3d34fbcc4 -redhat-image-scan-%: - docker pull ${IMAGE_BASE}_$*:${VERSION} +# +scan= +redhat-image-scan: + docker pull ${IMAGE_NAME}:${VERSION} source ~/.config/seldon/seldon-core/redhat-image-passwords.sh && \ - echo $${rh_password_xgboostserver_$*} | docker login -u unused scan.connect.redhat.com --password-stdin - docker tag ${IMAGE_BASE}_$*:${VERSION} scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} - docker push scan.connect.redhat.com/${scan_$*}/${IMAGE_NAME_BASE}_$*:${VERSION} + echo $${rh_password_xgboostserver} | docker login -u unused scan.connect.redhat.com --password-stdin + docker tag ${IMAGE_NAME}:${VERSION} scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} + docker push scan.connect.redhat.com/${scan}/${IMAGE_NAME_BASE}:${VERSION} -.PHONY: redhat-image-scan -redhat-image-scan: redhat-image-scan-rest redhat-image-scan-grpc diff --git a/servers/xgboostserver/environment_grpc b/servers/xgboostserver/environment similarity index 80% rename from servers/xgboostserver/environment_grpc rename to servers/xgboostserver/environment index 597312b3f9..de8df4b4af 100644 --- a/servers/xgboostserver/environment_grpc +++ b/servers/xgboostserver/environment @@ -1,4 +1,3 @@ MODEL_NAME=XGBoostServer -API_TYPE=GRPC SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/servers/xgboostserver/environment_rest b/servers/xgboostserver/environment_rest deleted file mode 100644 index 80af2fee92..0000000000 --- a/servers/xgboostserver/environment_rest +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=XGBoostServer -API_TYPE=REST -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/servers/xgboostserver/samples/iris.yaml b/servers/xgboostserver/samples/iris.yaml index b0fd0a614a..2af9f675e3 100644 --- a/servers/xgboostserver/samples/iris.yaml +++ b/servers/xgboostserver/samples/iris.yaml @@ -1,4 +1,4 @@ -apiVersion: machinelearning.seldon.io/v1alpha2 +apiVersion: machinelearning.seldon.io/v1 kind: SeldonDeployment metadata: name: xgboost @@ -11,4 +11,4 @@ spec: modelUri: gs://seldon-models/xgboost/iris name: classifier name: default - replicas: 1 + replicas: 1 \ No newline at end of file diff --git a/testing/docker/echo-model/Makefile b/testing/docker/echo-model/Makefile index ebcaa2b2b1..cd7a82b848 100644 --- a/testing/docker/echo-model/Makefile +++ b/testing/docker/echo-model/Makefile @@ -1,6 +1,6 @@ VERSION=0.1 IMAGE_BASE=seldonio/echo-model -S2I_IMAGE_VERSION=1.1.1-SNAPSHOT +S2I_IMAGE_VERSION := $(shell cat ../../../version.txt) build_image: s2i build \ diff --git a/testing/docker/fixed-model/Makefile b/testing/docker/fixed-model/Makefile index b56b5b6184..56b2159237 100644 --- a/testing/docker/fixed-model/Makefile +++ b/testing/docker/fixed-model/Makefile @@ -1,8 +1,8 @@ - +VERSION := $(shell cat ../../../version.txt) build_images: - s2i build -E environment_rest_v1 . seldonio/seldon-core-s2i-python3:0.15 seldonio/fixed-model:0.1 - s2i build -E environment_rest_v2 . seldonio/seldon-core-s2i-python3:0.15 seldonio/fixed-model:0.2 + s2i build -E environment_v1 . seldonio/seldon-core-s2i-python3:${VERSION} seldonio/fixed-model:0.1 + s2i build -E environment_v2 . seldonio/seldon-core-s2i-python3:${VERSION} seldonio/fixed-model:0.2 push_images: docker push seldonio/fixed-model:0.1 diff --git a/testing/docker/fixed-model/environment_rest_v1 b/testing/docker/fixed-model/environment_v1 similarity index 78% rename from testing/docker/fixed-model/environment_rest_v1 rename to testing/docker/fixed-model/environment_v1 index 85dd6d1826..d2e73399fe 100644 --- a/testing/docker/fixed-model/environment_rest_v1 +++ b/testing/docker/fixed-model/environment_v1 @@ -1,4 +1,3 @@ MODEL_NAME=ModelV1 -API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/testing/docker/fixed-model/environment_rest_v2 b/testing/docker/fixed-model/environment_v2 similarity index 78% rename from testing/docker/fixed-model/environment_rest_v2 rename to testing/docker/fixed-model/environment_v2 index 87dc775bd4..6a161be642 100644 --- a/testing/docker/fixed-model/environment_rest_v2 +++ b/testing/docker/fixed-model/environment_v2 @@ -1,4 +1,3 @@ MODEL_NAME=ModelV2 -API_TYPE=REST SERVICE_TYPE=MODEL PERSISTENCE=0 diff --git a/testing/resources/graph-tracing.json b/testing/resources/graph-tracing.json index 7ba756641a..60f646122d 100644 --- a/testing/resources/graph-tracing.json +++ b/testing/resources/graph-tracing.json @@ -9,8 +9,6 @@ }, "spec": { "name": "mymodel", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", "predictors": [ { "svcOrchSpec": { diff --git a/testing/resources/istio-ingressgateway-patch.yaml b/testing/resources/istio-ingressgateway-patch.yaml new file mode 100644 index 0000000000..d1478d4b65 --- /dev/null +++ b/testing/resources/istio-ingressgateway-patch.yaml @@ -0,0 +1,5 @@ +spec: + ports: + - name: http2 + nodePort: 31280 + port: 80 diff --git a/testing/resources/kind_config.yaml b/testing/resources/kind_config.yaml index 1f333337cc..69ddd90348 100644 --- a/testing/resources/kind_config.yaml +++ b/testing/resources/kind_config.yaml @@ -17,15 +17,15 @@ kubeadmConfigPatches: kubeReserved: cpu: "300m" memory: "300Mi" - ephemeral-storage: "1Gi" + ephemeral-storage: "10Gi" kubeReservedCgroup: "/kube-reserved" systemReserved: cpu: "300m" memory: "300Mi" - ephemeral-storage: "1Gi" + ephemeral-storage: "10Gi" evictionHard: - memory.available: "200Mi" - nodefs.available: "10%" + memory.available: "1Mi" + nodefs.available: "0%" featureGates: DynamicKubeletConfig: true RotateKubeletServerCertificate: true diff --git a/testing/resources/s2i_python_combiner.json b/testing/resources/s2i_python_combiner.json index 6bbae4c48e..61b7618cb8 100644 --- a/testing/resources/s2i_python_combiner.json +++ b/testing/resources/s2i_python_combiner.json @@ -6,20 +6,18 @@ }, "spec": { "name": "mycombiner", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", "predictors": [ { "componentSpecs": [{ "spec": { "containers": [ { - "image": "seldonio/testmodel_rest:0.1", + "image": "seldonio/testmodel:0.1", "imagePullPolicy": "Never", "name": "model" }, { - "image": "seldonio/testcombiner_rest:0.1", + "image": "seldonio/testcombiner:0.1", "imagePullPolicy": "Never", "name": "combiner" } @@ -29,18 +27,12 @@ }], "graph": { "children": [{ - "children": [], - "name": "model", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" + "children": [], + "name": "model", + "type": "MODEL" }], "name": "combiner", - "endpoint": { - "type" : "REST" - }, - "type": "COMBINER" + "type": "COMBINER" }, "name": "mycombiner", "replicas": 1 diff --git a/testing/resources/s2i_python_model.json b/testing/resources/s2i_python_model.json index 0fcbaa2cd3..31e99156d3 100644 --- a/testing/resources/s2i_python_model.json +++ b/testing/resources/s2i_python_model.json @@ -6,15 +6,13 @@ }, "spec": { "name": "mymodel", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", "predictors": [ { "componentSpecs": [{ "spec": { "containers": [ { - "image": "seldonio/testmodel_rest:0.1", + "image": "seldonio/testmodel:0.1", "imagePullPolicy": "Never", "name": "model" } @@ -25,10 +23,7 @@ "graph": { "children": [], "name": "model", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" + "type": "MODEL" }, "name": "mymodel", "replicas": 1 diff --git a/testing/resources/s2i_python_model_non200.json b/testing/resources/s2i_python_model_non200.json index 4d5286db02..7128e4527e 100644 --- a/testing/resources/s2i_python_model_non200.json +++ b/testing/resources/s2i_python_model_non200.json @@ -6,15 +6,13 @@ }, "spec": { "name": "mymodel", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", "predictors": [ { "componentSpecs": [{ "spec": { "containers": [ { - "image": "seldonio/testmodel_rest_non200:0.1", + "image": "seldonio/testmodel_non200:0.1", "imagePullPolicy": "Never", "name": "model" } @@ -25,9 +23,6 @@ "graph": { "children": [], "name": "model", - "endpoint": { - "type" : "REST" - }, "type": "MODEL" }, "name": "mymodel", diff --git a/testing/resources/s2i_python_output_transformer.json b/testing/resources/s2i_python_output_transformer.json index b2f95d71cc..a89e1f2e59 100644 --- a/testing/resources/s2i_python_output_transformer.json +++ b/testing/resources/s2i_python_output_transformer.json @@ -6,20 +6,18 @@ }, "spec": { "name": "mytrans", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", "predictors": [ { "componentSpecs": [{ "spec": { "containers": [ { - "image": "seldonio/testmodel_rest:0.1", + "image": "seldonio/testmodel:0.1", "imagePullPolicy": "Never", "name": "model" }, { - "image": "seldonio/testtransformer_rest:0.1", + "image": "seldonio/testtransformer:0.1", "imagePullPolicy": "Never", "name": "transformer" } @@ -31,16 +29,10 @@ "children": [{ "children": [], "name": "model", - "endpoint": { - "type" : "REST" - }, "type": "MODEL" }], "name": "transformer", - "endpoint": { - "type" : "REST" - }, - "type": "OUTPUT_TRANSFORMER" + "type": "OUTPUT_TRANSFORMER" }, "name": "mytrans", "replicas": 1 diff --git a/testing/resources/s2i_python_router.json b/testing/resources/s2i_python_router.json index 26094b0321..03ae586b92 100644 --- a/testing/resources/s2i_python_router.json +++ b/testing/resources/s2i_python_router.json @@ -6,20 +6,18 @@ }, "spec": { "name": "myrouter", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", "predictors": [ { "componentSpecs": [{ "spec": { "containers": [ { - "image": "seldonio/testmodel_rest:0.1", + "image": "seldonio/testmodel:0.1", "imagePullPolicy": "Never", "name": "model" }, { - "image": "seldonio/testrouter_rest:0.1", + "image": "seldonio/testrouter:0.1", "imagePullPolicy": "Never", "name": "router" } @@ -31,15 +29,9 @@ "children": [{ "children": [], "name": "model", - "endpoint": { - "type" : "REST" - }, "type": "MODEL" }], "name": "router", - "endpoint": { - "type" : "REST" - }, "type": "ROUTER" }, "name": "myrouter", diff --git a/testing/resources/s2i_python_transformer.json b/testing/resources/s2i_python_transformer.json index 263dbe63cb..dbba262f0f 100644 --- a/testing/resources/s2i_python_transformer.json +++ b/testing/resources/s2i_python_transformer.json @@ -6,15 +6,13 @@ }, "spec": { "name": "mytrans", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", "predictors": [ { "componentSpecs": [{ "spec": { "containers": [ { - "image": "seldonio/testtransformer_rest:0.1", + "image": "seldonio/testtransformer:0.1", "imagePullPolicy": "Never", "name": "trans" } @@ -25,9 +23,6 @@ "graph": { "children": [], "name": "trans", - "endpoint": { - "type" : "REST" - }, "type": "TRANSFORMER" }, "name": "mytrans", diff --git a/testing/s2i/python/combiner/environment_rest b/testing/s2i/python/combiner/environment similarity index 80% rename from testing/s2i/python/combiner/environment_rest rename to testing/s2i/python/combiner/environment index b073e78748..41d5df59df 100644 --- a/testing/s2i/python/combiner/environment_rest +++ b/testing/s2i/python/combiner/environment @@ -1,4 +1,3 @@ MODEL_NAME=MyCombiner -API_TYPE=REST SERVICE_TYPE=COMBINER PERSISTENCE=0 diff --git a/testing/s2i/python/combiner/environment_grpc b/testing/s2i/python/combiner/environment_grpc deleted file mode 100644 index d4a2ddbfe0..0000000000 --- a/testing/s2i/python/combiner/environment_grpc +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=MyCombiner -API_TYPE=GRPC -SERVICE_TYPE=COMBINER -PERSISTENCE=0 diff --git a/testing/s2i/python/model/environment_rest b/testing/s2i/python/model/environment similarity index 100% rename from testing/s2i/python/model/environment_rest rename to testing/s2i/python/model/environment diff --git a/testing/s2i/python/model/environment_grpc b/testing/s2i/python/model/environment_grpc deleted file mode 100644 index 12bdeaf558..0000000000 --- a/testing/s2i/python/model/environment_grpc +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=MyModel -API_TYPE=GRPC -SERVICE_TYPE=MODEL -PERSISTENCE=0 diff --git a/testing/s2i/python/model/environment_rest_non200 b/testing/s2i/python/model/environment_non200 similarity index 100% rename from testing/s2i/python/model/environment_rest_non200 rename to testing/s2i/python/model/environment_non200 diff --git a/testing/s2i/python/router/environment_grpc b/testing/s2i/python/router/environment similarity index 79% rename from testing/s2i/python/router/environment_grpc rename to testing/s2i/python/router/environment index 02b261788f..c47a09aae6 100644 --- a/testing/s2i/python/router/environment_grpc +++ b/testing/s2i/python/router/environment @@ -1,4 +1,3 @@ MODEL_NAME=MyRouter -API_TYPE=GRPC SERVICE_TYPE=ROUTER PERSISTENCE=0 diff --git a/testing/s2i/python/router/environment_rest b/testing/s2i/python/router/environment_rest deleted file mode 100644 index 982edc6b96..0000000000 --- a/testing/s2i/python/router/environment_rest +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=MyRouter -API_TYPE=REST -SERVICE_TYPE=ROUTER -PERSISTENCE=0 diff --git a/testing/s2i/python/transformer/environment_grpc b/testing/s2i/python/transformer/environment similarity index 82% rename from testing/s2i/python/transformer/environment_grpc rename to testing/s2i/python/transformer/environment index 5c0c12a084..dca7e0d9de 100644 --- a/testing/s2i/python/transformer/environment_grpc +++ b/testing/s2i/python/transformer/environment @@ -1,4 +1,3 @@ MODEL_NAME=MyTransformer -API_TYPE=GRPC SERVICE_TYPE=TRANSFORMER PERSISTENCE=0 diff --git a/testing/s2i/python/transformer/environment_rest b/testing/s2i/python/transformer/environment_rest deleted file mode 100644 index 56492d7069..0000000000 --- a/testing/s2i/python/transformer/environment_rest +++ /dev/null @@ -1,4 +0,0 @@ -MODEL_NAME=MyTransformer -API_TYPE=REST -SERVICE_TYPE=TRANSFORMER -PERSISTENCE=0 diff --git a/testing/scripts/.gitignore b/testing/scripts/.gitignore new file mode 100644 index 0000000000..525802c2c2 --- /dev/null +++ b/testing/scripts/.gitignore @@ -0,0 +1 @@ +istio-1.6.8 \ No newline at end of file diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 153b45b98d..23991a1b1d 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -4,44 +4,58 @@ VERSION := $(shell cat ../../version.txt) PYTEST_WORKERS ?= "4" SELDON_E2E_TESTS_USE_EXECUTOR ?= "true" +.PHONY: kind_create_cluster kind_create_cluster: kind create cluster --config ../resources/kind_config.yaml +.PHONY: kind_delete_cluster kind_delete_cluster: kind delete cluster +.PHONY: kind_build_engine kind_build_engine: cd ../../engine/ && make build_image kind load docker-image seldonio/engine:$(VERSION) --loglevel trace +.PHONY: kind_build_operator kind_build_operator: cd ../../operator && make kind-image-install +.PHONY: kind_build_executor kind_build_executor: cd ../../executor && make kind-image-install +.PHONY: kind_build_test_models kind_build_test_models: + make -C ../../examples/models/mean_classifier kind_load make -C ../docker/fixed-model kind_load_images make -C ../docker/echo-model kind_load_image +.PHONY: kind_build_prepackaged kind_build_prepackaged: cd ../../servers/sklearnserver && make kind_load cd ../../servers/xgboostserver && make kind_load cd ../../servers/mlflowserver && make kind_load - cd ../../integrations/tfserving && make kind_load + cd ../../servers/tfserving_proxy && make kind_load +.PHONY: kind_build_alibi_explain kind_build_alibi_explain: cd ../../components/alibi-explain-server && make kind_load +.PHONY: kind_build_alibi_detect kind_build_alibi_detect: cd ../../components/alibi-detect-server && make kind_load +.PHONY: kind_build_misc kind_build_misc: cd ../../components/seldon-request-logger && make kind_load cd ../../components/storage-initializer && make kind_load + cd ../../components/routers/epsilon-greedy && make kind_load +.PHONY: kind_build_images kind_build_images: build_protos kind_build_engine kind_build_operator kind_build_executor kind_build_test_models kind_build_prepackaged kind_build_alibi_explain kind_build_alibi_detect kind_build_misc +.PHONY: helm_setup helm_setup: helm repo add stable https://kubernetes-charts.storage.googleapis.com/ helm repo add seldonio https://storage.googleapis.com/seldon-charts @@ -49,6 +63,7 @@ helm_setup: helm repo add datawire https://www.getambassador.io helm repo update +.PHONY: install_ambassador install_ambassador: helm upgrade ambassador \ datawire/ambassador \ @@ -60,6 +75,7 @@ install_ambassador: --set replicaCount=1 \ --wait --install +.PHONY: install_jaeger install_jaeger: helm upgrade jaeger-operator \ jaegertracing/jaeger-operator \ @@ -69,11 +85,12 @@ install_jaeger: kubectl apply -f ../resources/jaeger.yaml --namespace seldon -install_cert_manager: - cd ../../operator && make install-cert-manager - sleep 5 #https://github.com/jetstack/cert-manager/issues/2273 +#install_cert_manager: +# cd ../../operator && make install-cert-manager +# sleep 5 #https://github.com/jetstack/cert-manager/issues/2273 -install_seldon: install_cert_manager +.PHONY: install_seldon +install_seldon: kubectl create namespace seldon-system || echo "namespace seldon-system exists" helm delete seldon --namespace seldon-system || echo "seldon-core not installed" helm upgrade seldon \ @@ -86,35 +103,43 @@ install_seldon: install_cert_manager --set executor.enabled=$(SELDON_E2E_TESTS_USE_EXECUTOR) \ --wait --install +# Install istio and patch nodePort to allow 8004 to point to ingress gateway +.PHONY: install_istio install_istio: - kubectl apply -f istio-1.4.2.yaml - kubectl rollout status deployment.apps/istio-ingressgateway -n istio-system - kubectl rollout status deployment.apps/istio-pilot -n istio-system - kubectl rollout status deployment.apps/istio-citadel -n istio-system + curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.6.8 TARGET_ARCH=x86_64 sh - + istio-1.6.8/bin/istioctl install --set profile=demo + kubectl patch svc istio-ingressgateway -n istio-system --patch "$$(cat ../resources/istio-ingressgateway-patch.yaml)" kubectl create -f ../resources/seldon-gateway.yaml -n istio-system +.PHONY: install_keda install_keda: kubectl apply -f https://github.com/kedacore/keda/releases/download/v2.0.0-beta/keda-2.0.0-beta.yaml +.PHONY: create_namespaces create_namespaces: kubectl create namespace seldon || echo "Namespace seldon already exists" kubectl create namespace test1 || echo "Namespace test1 already exists" +.PHONY: set_namespace set_namespace: kubectl config set-context $$(kubectl config current-context) --namespace=seldon +.PHONY: install_metrics # Derived from https://github.com/kubernetes-sigs/kind/issues/398#issuecomment-478325705 install_metrics: kubectl apply -f metrics.yaml +.PHONY: kind_setup kind_setup: create_namespaces helm_setup install_ambassador install_istio install_keda install_jaeger install_seldon set_namespace install_metrics + port-forward-ambassador: kubectl port-forward $$(kubectl get pods -n seldon -l app.kubernetes.io/name=ambassador -o jsonpath='{.items[0].metadata.name}') -n seldon 8003:8080 - +.PHONY: s2i_build_base_images s2i_build_base_images: cd ../../wrappers/s2i/python/build_scripts && ./build_all_local.sh +.PHONY: build_protos build_protos: cp ../../proto/prediction.proto ./proto cd ../../proto/tensorflow && make create_protos @@ -130,8 +155,8 @@ install: pip install -r dev_requirements.txt pip install -e ../../python +# Run the core tests in parallel test_parallel: build_protos install - # Run the core tests in parallel pytest \ --verbose \ -s \ @@ -139,8 +164,8 @@ test_parallel: build_protos install -n $(PYTEST_WORKERS) \ -m "not sequential and not notebooks" 2>&1 +# Then run the s2i tests in sequence (as they currently fail in parallel test_sequential: build_protos install - # Then run the s2i tests in sequence (as they currently fail in parallel pytest \ --verbose \ -s \ diff --git a/testing/scripts/istio-1.4.2.yaml b/testing/scripts/istio-1.4.2.yaml deleted file mode 100644 index 62467f991e..0000000000 --- a/testing/scripts/istio-1.4.2.yaml +++ /dev/null @@ -1,24825 +0,0 @@ -# Resources for Base component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-reader-istio-system - labels: - app: istio-reader - release: istio -rules: -- apiGroups: - - "config.istio.io" - - "rbac.istio.io" - - "security.istio.io" - - "networking.istio.io" - - "authentication.istio.io" - resources: ["*"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["endpoints", "pods", "services", "nodes", "replicationcontrollers"] - verbs: ["get", "list", "watch"] -- apiGroups: ["apps"] - resources: ["replicasets"] - verbs: ["get", "list", "watch"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: istio-reader-istio-system - labels: - app: istio-reader - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: istio-reader-istio-system -subjects: - - kind: ServiceAccount - name: istio-reader-service-account - namespace: istio-system ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: mixer - chart: istio - heritage: Tiller - istio: core - package: istio.io.mixer - release: istio - name: attributemanifests.config.istio.io -spec: - group: config.istio.io - names: - categories: - - istio-io - - policy-istio-io - kind: attributemanifest - plural: attributemanifests - singular: attributemanifest - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Describes the rules used to configure Mixer''s policy and - telemetry features. See more details at: https://istio.io/docs/reference/config/policy-and-telemetry/istio.policy.v1beta1.html' - properties: - attributes: - additionalProperties: - properties: - description: - description: A human-readable description of the attribute's purpose. - format: string - type: string - valueType: - description: The type of data carried by this attribute. - enum: - - VALUE_TYPE_UNSPECIFIED - - STRING - - INT64 - - DOUBLE - - BOOL - - TIMESTAMP - - IP_ADDRESS - - EMAIL_ADDRESS - - URI - - DNS_NAME - - DURATION - - STRING_MAP - type: string - type: object - description: The set of attributes this Istio component will be responsible - for producing at runtime. - type: object - name: - description: Name of the component producing these attributes. - format: string - type: string - revision: - description: The revision of this document. - format: string - type: string - type: object - type: object - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - heritage: Tiller - istio: rbac - release: istio - name: clusterrbacconfigs.rbac.istio.io -spec: - group: rbac.istio.io - names: - categories: - - istio-io - - rbac-istio-io - kind: ClusterRbacConfig - plural: clusterrbacconfigs - singular: clusterrbacconfig - scope: Cluster - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration for Role Based Access Control. See more details - at: https://istio.io/docs/reference/config/authorization/istio.rbac.v1alpha1.html' - properties: - enforcementMode: - enum: - - ENFORCED - - PERMISSIVE - type: string - exclusion: - description: A list of services or namespaces that should not be enforced - by Istio RBAC policies. - properties: - namespaces: - description: A list of namespaces. - items: - format: string - type: string - type: array - services: - description: A list of services. - items: - format: string - type: string - type: array - type: object - inclusion: - description: A list of services or namespaces that should be enforced - by Istio RBAC policies. - properties: - namespaces: - description: A list of namespaces. - items: - format: string - type: string - type: array - services: - description: A list of services. - items: - format: string - type: string - type: array - type: object - mode: - description: Istio RBAC mode. - enum: - - "OFF" - - "ON" - - ON_WITH_INCLUSION - - ON_WITH_EXCLUSION - type: string - type: object - type: object - versions: - - name: v1alpha1 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: destinationrules.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .spec.host - description: The name of a service from the service registry - name: Host - type: string - - JSONPath: .metadata.creationTimestamp - description: |- - CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. - Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata - name: Age - type: date - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: DestinationRule - listKind: DestinationRuleList - plural: destinationrules - shortNames: - - dr - singular: destinationrule - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting load balancing, outlier detection, - etc. See more details at: https://istio.io/docs/reference/config/networking/v1alpha3/destination-rule.html' - properties: - exportTo: - description: A list of namespaces to which this destination rule is - exported. - items: - format: string - type: string - type: array - host: - description: The name of a service from the service registry. - format: string - type: string - subsets: - items: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - name: - description: Name of the subset. - format: string - type: string - trafficPolicy: - description: Traffic policies that apply to this subset. - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests - to a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection - to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - type: object - tcp: - description: Settings common to both HTTP and TCP upstream - connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutiveErrors: - format: int32 - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - portLevelSettings: - description: Traffic policies specific to individual ports. - items: - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP - requests to a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a - backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per - connection to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - type: object - tcp: - description: Settings common to both HTTP and TCP - upstream connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on - the socket to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer - algorithms. - oneOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutiveErrors: - format: int32 - type: integer - interval: - description: Time interval between ejection sweep - analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - port: - properties: - number: - type: integer - type: object - tls: - description: TLS related settings for connections to - the upstream service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server - during TLS handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: array - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during - TLS handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: object - type: array - trafficPolicy: - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should be upgraded - to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests to - a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection pool - connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection to - a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - type: object - tcp: - description: Settings common to both HTTP and TCP upstream connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections to - a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutiveErrors: - format: int32 - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - portLevelSettings: - description: Traffic policies specific to individual ports. - items: - properties: - connectionPool: - properties: - http: - description: HTTP connection pool settings. - properties: - h2UpgradePolicy: - description: Specify if http1.1 connection should - be upgraded to http2 for the associated destination. - enum: - - DEFAULT - - DO_NOT_UPGRADE - - UPGRADE - type: string - http1MaxPendingRequests: - description: Maximum number of pending HTTP requests - to a destination. - format: int32 - type: integer - http2MaxRequests: - description: Maximum number of requests to a backend. - format: int32 - type: integer - idleTimeout: - description: The idle timeout for upstream connection - pool connections. - type: string - maxRequestsPerConnection: - description: Maximum number of requests per connection - to a backend. - format: int32 - type: integer - maxRetries: - format: int32 - type: integer - type: object - tcp: - description: Settings common to both HTTP and TCP upstream - connections. - properties: - connectTimeout: - description: TCP connection timeout. - type: string - maxConnections: - description: Maximum number of HTTP1 /TCP connections - to a destination host. - format: int32 - type: integer - tcpKeepalive: - description: If set then set SO_KEEPALIVE on the socket - to enable TCP Keepalives. - properties: - interval: - description: The time duration between keep-alive - probes. - type: string - probes: - type: integer - time: - type: string - type: object - type: object - type: object - loadBalancer: - description: Settings controlling the load balancer algorithms. - oneOf: - - required: - - simple - - properties: - consistentHash: - oneOf: - - required: - - httpHeaderName - - required: - - httpCookie - - required: - - useSourceIp - required: - - consistentHash - properties: - consistentHash: - properties: - httpCookie: - description: Hash based on HTTP cookie. - properties: - name: - description: Name of the cookie. - format: string - type: string - path: - description: Path to set for the cookie. - format: string - type: string - ttl: - description: Lifetime of the cookie. - type: string - type: object - httpHeaderName: - description: Hash based on a specific HTTP header. - format: string - type: string - minimumRingSize: - type: integer - useSourceIp: - description: Hash based on the source IP address. - type: boolean - type: object - simple: - enum: - - ROUND_ROBIN - - LEAST_CONN - - RANDOM - - PASSTHROUGH - type: string - type: object - outlierDetection: - properties: - baseEjectionTime: - description: Minimum ejection duration. - type: string - consecutiveErrors: - format: int32 - type: integer - interval: - description: Time interval between ejection sweep analysis. - type: string - maxEjectionPercent: - format: int32 - type: integer - minHealthPercent: - format: int32 - type: integer - type: object - port: - properties: - number: - type: integer - type: object - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during - TLS handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: array - tls: - description: TLS related settings for connections to the upstream - service. - properties: - caCertificates: - format: string - type: string - clientCertificate: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - mode: - enum: - - DISABLE - - SIMPLE - - MUTUAL - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - sni: - description: SNI string to present to the server during TLS - handshake. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - type: object - type: object - versions: - - name: v1alpha3 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: envoyfilters.networking.istio.io -spec: - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: EnvoyFilter - plural: envoyfilters - singular: envoyfilter - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Customizing Envoy configuration generated by Istio. See more - details at: https://istio.io/docs/reference/config/networking/v1alpha3/envoy-filter.html' - properties: - configPatches: - description: One or more patches with match conditions. - items: - properties: - applyTo: - enum: - - INVALID - - LISTENER - - FILTER_CHAIN - - NETWORK_FILTER - - HTTP_FILTER - - ROUTE_CONFIGURATION - - VIRTUAL_HOST - - HTTP_ROUTE - - CLUSTER - type: string - match: - description: Match on listener/route configuration/cluster. - oneOf: - - required: - - listener - - required: - - routeConfiguration - - required: - - cluster - properties: - cluster: - description: Match on envoy cluster attributes. - properties: - name: - description: The exact name of the cluster to match. - format: string - type: string - portNumber: - description: The service port for which this cluster was - generated. - type: integer - service: - description: The fully qualified service name for this - cluster. - format: string - type: string - subset: - description: The subset associated with the service. - format: string - type: string - type: object - context: - description: The specific config generation context to match - on. - enum: - - ANY - - SIDECAR_INBOUND - - SIDECAR_OUTBOUND - - GATEWAY - type: string - listener: - description: Match on envoy listener attributes. - properties: - filterChain: - description: Match a specific filter chain in a listener. - properties: - applicationProtocols: - description: Applies only to sidecars. - format: string - type: string - filter: - description: The name of a specific filter to apply - the patch to. - properties: - name: - description: The filter name to match on. - format: string - type: string - subFilter: - properties: - name: - description: The filter name to match on. - format: string - type: string - type: object - type: object - name: - description: The name assigned to the filter chain. - format: string - type: string - sni: - description: The SNI value used by a filter chain's - match condition. - format: string - type: string - transportProtocol: - description: Applies only to SIDECAR_INBOUND context. - format: string - type: string - type: object - name: - description: Match a specific listener by its name. - format: string - type: string - portName: - format: string - type: string - portNumber: - type: integer - type: object - proxy: - description: Match on properties associated with a proxy. - properties: - metadata: - additionalProperties: - format: string - type: string - type: object - proxyVersion: - format: string - type: string - type: object - routeConfiguration: - description: Match on envoy HTTP route configuration attributes. - properties: - gateway: - format: string - type: string - name: - description: Route configuration name to match on. - format: string - type: string - portName: - description: Applicable only for GATEWAY context. - format: string - type: string - portNumber: - type: integer - vhost: - properties: - name: - format: string - type: string - route: - description: Match a specific route within the virtual - host. - properties: - action: - description: Match a route with specific action - type. - enum: - - ANY - - ROUTE - - REDIRECT - - DIRECT_RESPONSE - type: string - name: - format: string - type: string - type: object - type: object - type: object - type: object - patch: - description: The patch to apply along with the operation. - properties: - operation: - description: Determines how the patch should be applied. - enum: - - INVALID - - MERGE - - ADD - - REMOVE - - INSERT_BEFORE - - INSERT_AFTER - type: string - value: - description: The JSON config of the object being patched. - type: object - type: object - type: object - type: array - filters: - items: - properties: - filterConfig: - type: object - filterName: - description: The name of the filter to instantiate. - format: string - type: string - filterType: - description: The type of filter to instantiate. - enum: - - INVALID - - HTTP - - NETWORK - type: string - insertPosition: - description: Insert position in the filter chain. - properties: - index: - description: Position of this filter in the filter chain. - enum: - - FIRST - - LAST - - BEFORE - - AFTER - type: string - relativeTo: - format: string - type: string - type: object - listenerMatch: - properties: - address: - description: One or more IP addresses to which the listener - is bound. - items: - format: string - type: string - type: array - listenerProtocol: - description: Selects a class of listeners for the same protocol. - enum: - - ALL - - HTTP - - TCP - type: string - listenerType: - description: Inbound vs outbound sidecar listener or gateway - listener. - enum: - - ANY - - SIDECAR_INBOUND - - SIDECAR_OUTBOUND - - GATEWAY - type: string - portNamePrefix: - format: string - type: string - portNumber: - type: integer - type: object - type: object - type: array - workloadLabels: - additionalProperties: - format: string - type: string - description: Deprecated. - type: object - workloadSelector: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - type: object - versions: - - name: v1alpha3 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: gateways.networking.istio.io -spec: - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: Gateway - plural: gateways - shortNames: - - gw - singular: gateway - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting edge load balancer. See more details - at: https://istio.io/docs/reference/config/networking/v1alpha3/gateway.html' - properties: - selector: - additionalProperties: - format: string - type: string - type: object - servers: - description: A list of server specifications. - items: - properties: - bind: - format: string - type: string - defaultEndpoint: - format: string - type: string - hosts: - description: One or more hosts exposed by this gateway. - items: - format: string - type: string - type: array - port: - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - type: object - tls: - description: Set of TLS related options that govern the server's - behavior. - properties: - caCertificates: - description: REQUIRED if mode is `MUTUAL`. - format: string - type: string - cipherSuites: - description: 'Optional: If specified, only support the specified - cipher list.' - items: - format: string - type: string - type: array - credentialName: - format: string - type: string - httpsRedirect: - type: boolean - maxProtocolVersion: - description: 'Optional: Maximum TLS protocol version.' - enum: - - TLS_AUTO - - TLSV1_0 - - TLSV1_1 - - TLSV1_2 - - TLSV1_3 - type: string - minProtocolVersion: - description: 'Optional: Minimum TLS protocol version.' - enum: - - TLS_AUTO - - TLSV1_0 - - TLSV1_1 - - TLSV1_2 - - TLSV1_3 - type: string - mode: - enum: - - PASSTHROUGH - - SIMPLE - - MUTUAL - - AUTO_PASSTHROUGH - - ISTIO_MUTUAL - type: string - privateKey: - description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. - format: string - type: string - serverCertificate: - description: REQUIRED if mode is `SIMPLE` or `MUTUAL`. - format: string - type: string - subjectAltNames: - items: - format: string - type: string - type: array - verifyCertificateHash: - items: - format: string - type: string - type: array - verifyCertificateSpki: - items: - format: string - type: string - type: array - type: object - type: object - type: array - type: object - type: object - versions: - - name: v1alpha3 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-mixer - chart: istio - heritage: Tiller - release: istio - name: httpapispecbindings.config.istio.io -spec: - group: config.istio.io - names: - categories: - - istio-io - - apim-istio-io - kind: HTTPAPISpecBinding - plural: httpapispecbindings - singular: httpapispecbinding - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - properties: - api_specs: - items: - properties: - name: - description: The short name of the HTTPAPISpec. - format: string - type: string - namespace: - description: Optional namespace of the HTTPAPISpec. - format: string - type: string - type: object - type: array - apiSpecs: - items: - properties: - name: - description: The short name of the HTTPAPISpec. - format: string - type: string - namespace: - description: Optional namespace of the HTTPAPISpec. - format: string - type: string - type: object - type: array - services: - description: One or more services to map the listed HTTPAPISpec onto. - items: - properties: - domain: - description: Domain suffix used to construct the service FQDN - in implementations that support such specification. - format: string - type: string - labels: - additionalProperties: - format: string - type: string - description: Optional one or more labels that uniquely identify - the service version. - type: object - name: - description: The short name of the service such as "foo". - format: string - type: string - namespace: - description: Optional namespace of the service. - format: string - type: string - service: - description: The service FQDN. - format: string - type: string - type: object - type: array - type: object - type: object - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-mixer - chart: istio - heritage: Tiller - release: istio - name: httpapispecs.config.istio.io -spec: - group: config.istio.io - names: - categories: - - istio-io - - apim-istio-io - kind: HTTPAPISpec - plural: httpapispecs - singular: httpapispec - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - properties: - api_keys: - items: - oneOf: - - required: - - query - - required: - - header - - required: - - cookie - properties: - cookie: - format: string - type: string - header: - description: API key is sent in a request header. - format: string - type: string - query: - description: API Key is sent as a query parameter. - format: string - type: string - type: object - type: array - apiKeys: - items: - oneOf: - - required: - - query - - required: - - header - - required: - - cookie - properties: - cookie: - format: string - type: string - header: - description: API key is sent in a request header. - format: string - type: string - query: - description: API Key is sent as a query parameter. - format: string - type: string - type: object - type: array - attributes: - properties: - attributes: - additionalProperties: - oneOf: - - required: - - stringValue - - required: - - int64Value - - required: - - doubleValue - - required: - - boolValue - - required: - - bytesValue - - required: - - timestampValue - - required: - - durationValue - - required: - - stringMapValue - properties: - boolValue: - type: boolean - bytesValue: - format: binary - type: string - doubleValue: - format: double - type: number - durationValue: - type: string - int64Value: - format: int64 - type: integer - stringMapValue: - properties: - entries: - additionalProperties: - format: string - type: string - description: Holds a set of name/value pairs. - type: object - type: object - stringValue: - format: string - type: string - timestampValue: - format: dateTime - type: string - type: object - description: A map of attribute name to its value. - type: object - type: object - patterns: - description: List of HTTP patterns to match. - items: - oneOf: - - required: - - uriTemplate - - required: - - regex - properties: - attributes: - properties: - attributes: - additionalProperties: - oneOf: - - required: - - stringValue - - required: - - int64Value - - required: - - doubleValue - - required: - - boolValue - - required: - - bytesValue - - required: - - timestampValue - - required: - - durationValue - - required: - - stringMapValue - properties: - boolValue: - type: boolean - bytesValue: - format: binary - type: string - doubleValue: - format: double - type: number - durationValue: - type: string - int64Value: - format: int64 - type: integer - stringMapValue: - properties: - entries: - additionalProperties: - format: string - type: string - description: Holds a set of name/value pairs. - type: object - type: object - stringValue: - format: string - type: string - timestampValue: - format: dateTime - type: string - type: object - description: A map of attribute name to its value. - type: object - type: object - httpMethod: - format: string - type: string - regex: - format: string - type: string - uriTemplate: - format: string - type: string - type: object - type: array - type: object - type: object - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-citadel - chart: istio - heritage: Tiller - release: istio - name: meshpolicies.authentication.istio.io -spec: - group: authentication.istio.io - names: - categories: - - istio-io - - authentication-istio-io - kind: MeshPolicy - listKind: MeshPolicyList - plural: meshpolicies - singular: meshpolicy - scope: Cluster - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Authentication policy for Istio services. See more details - at: https://istio.io/docs/reference/config/istio.authentication.v1alpha1.html' - properties: - originIsOptional: - type: boolean - origins: - description: List of authentication methods that can be used for origin - authentication. - items: - properties: - jwt: - description: Jwt params for the method. - properties: - audiences: - items: - format: string - type: string - type: array - issuer: - description: Identifies the issuer that issued the JWT. - format: string - type: string - jwks: - description: JSON Web Key Set of public keys to validate signature - of the JWT. - format: string - type: string - jwks_uri: - format: string - type: string - jwksUri: - format: string - type: string - jwt_headers: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtHeaders: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtParams: - description: JWT is sent in a query parameter. - items: - format: string - type: string - type: array - trigger_rules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - triggerRules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - type: object - type: object - type: array - peerIsOptional: - type: boolean - peers: - description: List of authentication methods that can be used for peer - authentication. - items: - oneOf: - - required: - - mtls - - required: - - jwt - properties: - jwt: - properties: - audiences: - items: - format: string - type: string - type: array - issuer: - description: Identifies the issuer that issued the JWT. - format: string - type: string - jwks: - description: JSON Web Key Set of public keys to validate signature - of the JWT. - format: string - type: string - jwks_uri: - format: string - type: string - jwksUri: - format: string - type: string - jwt_headers: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtHeaders: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtParams: - description: JWT is sent in a query parameter. - items: - format: string - type: string - type: array - trigger_rules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - triggerRules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - type: object - mtls: - description: Set if mTLS is used. - properties: - allowTls: - description: WILL BE DEPRECATED, if set, will translates to - `TLS_PERMISSIVE` mode. - type: boolean - mode: - description: Defines the mode of mTLS authentication. - enum: - - STRICT - - PERMISSIVE - type: string - type: object - type: object - type: array - principalBinding: - description: Define whether peer or origin identity should be use for - principal. - enum: - - USE_PEER - - USE_ORIGIN - type: string - targets: - description: List rules to select workloads that the policy should be - applied on. - items: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - name: - description: The name must be a short name from the service registry. - format: string - type: string - ports: - description: Specifies the ports. - items: - oneOf: - - required: - - number - - required: - - name - properties: - name: - format: string - type: string - number: - type: integer - type: object - type: array - type: object - type: array - type: object - type: object - versions: - - name: v1alpha1 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-citadel - chart: istio - heritage: Tiller - release: istio - name: policies.authentication.istio.io -spec: - group: authentication.istio.io - names: - categories: - - istio-io - - authentication-istio-io - kind: Policy - plural: policies - singular: policy - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Authentication policy for Istio services. See more details - at: https://istio.io/docs/reference/config/istio.authentication.v1alpha1.html' - properties: - originIsOptional: - type: boolean - origins: - description: List of authentication methods that can be used for origin - authentication. - items: - properties: - jwt: - description: Jwt params for the method. - properties: - audiences: - items: - format: string - type: string - type: array - issuer: - description: Identifies the issuer that issued the JWT. - format: string - type: string - jwks: - description: JSON Web Key Set of public keys to validate signature - of the JWT. - format: string - type: string - jwks_uri: - format: string - type: string - jwksUri: - format: string - type: string - jwt_headers: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtHeaders: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtParams: - description: JWT is sent in a query parameter. - items: - format: string - type: string - type: array - trigger_rules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - triggerRules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - type: object - type: object - type: array - peerIsOptional: - type: boolean - peers: - description: List of authentication methods that can be used for peer - authentication. - items: - oneOf: - - required: - - mtls - - required: - - jwt - properties: - jwt: - properties: - audiences: - items: - format: string - type: string - type: array - issuer: - description: Identifies the issuer that issued the JWT. - format: string - type: string - jwks: - description: JSON Web Key Set of public keys to validate signature - of the JWT. - format: string - type: string - jwks_uri: - format: string - type: string - jwksUri: - format: string - type: string - jwt_headers: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtHeaders: - description: JWT is sent in a request header. - items: - format: string - type: string - type: array - jwtParams: - description: JWT is sent in a query parameter. - items: - format: string - type: string - type: array - trigger_rules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - triggerRules: - items: - properties: - excluded_paths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - excludedPaths: - description: List of paths to be excluded from the request. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - included_paths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - includedPaths: - description: List of paths that the request must include. - items: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - suffix - - required: - - regex - properties: - exact: - description: exact string match. - format: string - type: string - prefix: - description: prefix-based match. - format: string - type: string - regex: - description: ECMAscript style regex-based match - as defined by [EDCA-262](http://en.cppreference.com/w/cpp/regex/ecmascript). - format: string - type: string - suffix: - description: suffix-based match. - format: string - type: string - type: object - type: array - type: object - type: array - type: object - mtls: - description: Set if mTLS is used. - properties: - allowTls: - description: WILL BE DEPRECATED, if set, will translates to - `TLS_PERMISSIVE` mode. - type: boolean - mode: - description: Defines the mode of mTLS authentication. - enum: - - STRICT - - PERMISSIVE - type: string - type: object - type: object - type: array - principalBinding: - description: Define whether peer or origin identity should be use for - principal. - enum: - - USE_PEER - - USE_ORIGIN - type: string - targets: - description: List rules to select workloads that the policy should be - applied on. - items: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - name: - description: The name must be a short name from the service registry. - format: string - type: string - ports: - description: Specifies the ports. - items: - oneOf: - - required: - - number - - required: - - name - properties: - name: - format: string - type: string - number: - type: integer - type: object - type: array - type: object - type: array - type: object - type: object - versions: - - name: v1alpha1 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-mixer - chart: istio - heritage: Tiller - release: istio - name: quotaspecbindings.config.istio.io -spec: - group: config.istio.io - names: - categories: - - istio-io - - apim-istio-io - kind: QuotaSpecBinding - plural: quotaspecbindings - singular: quotaspecbinding - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - properties: - quotaSpecs: - items: - properties: - name: - description: The short name of the QuotaSpec. - format: string - type: string - namespace: - description: Optional namespace of the QuotaSpec. - format: string - type: string - type: object - type: array - services: - description: One or more services to map the listed QuotaSpec onto. - items: - properties: - domain: - description: Domain suffix used to construct the service FQDN - in implementations that support such specification. - format: string - type: string - labels: - additionalProperties: - format: string - type: string - description: Optional one or more labels that uniquely identify - the service version. - type: object - name: - description: The short name of the service such as "foo". - format: string - type: string - namespace: - description: Optional namespace of the service. - format: string - type: string - service: - description: The service FQDN. - format: string - type: string - type: object - type: array - type: object - type: object - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-mixer - chart: istio - heritage: Tiller - release: istio - name: quotaspecs.config.istio.io -spec: - group: config.istio.io - names: - categories: - - istio-io - - apim-istio-io - kind: QuotaSpec - plural: quotaspecs - singular: quotaspec - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: Determines the quotas used for individual requests. - properties: - rules: - description: A list of Quota rules. - items: - properties: - match: - description: If empty, match all request. - items: - properties: - clause: - additionalProperties: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - format: string - type: string - type: object - description: Map of attribute names to StringMatch type. - type: object - type: object - type: array - quotas: - description: The list of quotas to charge. - items: - properties: - charge: - format: int32 - type: integer - quota: - format: string - type: string - type: object - type: array - type: object - type: array - type: object - type: object - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: mixer - chart: istio - heritage: Tiller - istio: rbac - package: istio.io.mixer - release: istio - name: rbacconfigs.rbac.istio.io -spec: - group: rbac.istio.io - names: - categories: - - istio-io - - rbac-istio-io - kind: RbacConfig - plural: rbacconfigs - singular: rbacconfig - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration for Role Based Access Control. See more details - at: https://istio.io/docs/reference/config/authorization/istio.rbac.v1alpha1.html' - properties: - enforcementMode: - enum: - - ENFORCED - - PERMISSIVE - type: string - exclusion: - description: A list of services or namespaces that should not be enforced - by Istio RBAC policies. - properties: - namespaces: - description: A list of namespaces. - items: - format: string - type: string - type: array - services: - description: A list of services. - items: - format: string - type: string - type: array - type: object - inclusion: - description: A list of services or namespaces that should be enforced - by Istio RBAC policies. - properties: - namespaces: - description: A list of namespaces. - items: - format: string - type: string - type: array - services: - description: A list of services. - items: - format: string - type: string - type: array - type: object - mode: - description: Istio RBAC mode. - enum: - - "OFF" - - "ON" - - ON_WITH_INCLUSION - - ON_WITH_EXCLUSION - type: string - type: object - type: object - versions: - - name: v1alpha1 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: mixer - chart: istio - heritage: Tiller - istio: core - package: istio.io.mixer - release: istio - name: rules.config.istio.io -spec: - group: config.istio.io - names: - categories: - - istio-io - - policy-istio-io - kind: rule - plural: rules - singular: rule - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Describes the rules used to configure Mixer''s policy and - telemetry features. See more details at: https://istio.io/docs/reference/config/policy-and-telemetry/istio.policy.v1beta1.html' - properties: - actions: - description: The actions that will be executed when match evaluates - to `true`. - items: - properties: - handler: - description: Fully qualified name of the handler to invoke. - format: string - type: string - instances: - items: - format: string - type: string - type: array - name: - description: A handle to refer to the results of the action. - format: string - type: string - type: object - type: array - match: - description: Match is an attribute based predicate. - format: string - type: string - requestHeaderOperations: - items: - properties: - name: - description: Header name literal value. - format: string - type: string - operation: - description: Header operation type. - enum: - - REPLACE - - REMOVE - - APPEND - type: string - values: - description: Header value expressions. - items: - format: string - type: string - type: array - type: object - type: array - responseHeaderOperations: - items: - properties: - name: - description: Header name literal value. - format: string - type: string - operation: - description: Header operation type. - enum: - - REPLACE - - REMOVE - - APPEND - type: string - values: - description: Header value expressions. - items: - format: string - type: string - type: array - type: object - type: array - sampling: - properties: - random: - description: Provides filtering of actions based on random selection - per request. - properties: - attributeExpression: - description: Specifies an attribute expression to use to override - the numerator in the `percent_sampled` field. - format: string - type: string - percentSampled: - description: The default sampling rate, expressed as a percentage. - properties: - denominator: - description: Specifies the denominator. - enum: - - HUNDRED - - TEN_THOUSAND - type: string - numerator: - description: Specifies the numerator. - type: integer - type: object - useIndependentRandomness: - description: By default sampling will be based on the value - of the request header `x-request-id`. - type: boolean - type: object - rateLimit: - properties: - maxUnsampledEntries: - description: Number of entries to allow during the `sampling_duration` - before sampling is enforced. - format: int64 - type: integer - samplingDuration: - description: Window in which to enforce the sampling rate. - type: string - samplingRate: - description: The rate at which to sample entries once the unsampled - limit has been reached. - format: int64 - type: integer - type: object - type: object - type: object - type: object - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: serviceentries.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .spec.hosts - description: The hosts associated with the ServiceEntry - name: Hosts - type: string - - JSONPath: .spec.location - description: Whether the service is external to the mesh or part of the mesh (MESH_EXTERNAL - or MESH_INTERNAL) - name: Location - type: string - - JSONPath: .spec.resolution - description: Service discovery mode for the hosts (NONE, STATIC, or DNS) - name: Resolution - type: string - - JSONPath: .metadata.creationTimestamp - description: |- - CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. - Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata - name: Age - type: date - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: ServiceEntry - listKind: ServiceEntryList - plural: serviceentries - shortNames: - - se - singular: serviceentry - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting service registry. See more details - at: https://istio.io/docs/reference/config/networking/v1alpha3/service-entry.html' - properties: - addresses: - description: The virtual IP addresses associated with the service. - items: - format: string - type: string - type: array - endpoints: - description: One or more endpoints associated with the service. - items: - properties: - address: - format: string - type: string - labels: - additionalProperties: - format: string - type: string - description: One or more labels associated with the endpoint. - type: object - locality: - description: The locality associated with the endpoint. - format: string - type: string - network: - format: string - type: string - ports: - additionalProperties: - type: integer - description: Set of ports associated with the endpoint. - type: object - weight: - description: The load balancing weight associated with the endpoint. - type: integer - type: object - type: array - exportTo: - description: A list of namespaces to which this service is exported. - items: - format: string - type: string - type: array - hosts: - description: The hosts associated with the ServiceEntry. - items: - format: string - type: string - type: array - location: - enum: - - MESH_EXTERNAL - - MESH_INTERNAL - type: string - ports: - description: The ports associated with the external service. - items: - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - type: object - type: array - resolution: - description: Service discovery mode for the hosts. - enum: - - NONE - - STATIC - - DNS - type: string - subjectAltNames: - items: - format: string - type: string - type: array - type: object - type: object - versions: - - name: v1alpha3 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: mixer - chart: istio - heritage: Tiller - istio: rbac - package: istio.io.mixer - release: istio - name: servicerolebindings.rbac.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .spec.roleRef.name - description: The name of the ServiceRole object being referenced - name: Reference - type: string - - JSONPath: .metadata.creationTimestamp - description: |- - CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. - Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata - name: Age - type: date - group: rbac.istio.io - names: - categories: - - istio-io - - rbac-istio-io - kind: ServiceRoleBinding - plural: servicerolebindings - singular: servicerolebinding - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration for Role Based Access Control. See more details - at: https://istio.io/docs/reference/config/authorization/istio.rbac.v1alpha1.html' - properties: - actions: - items: - properties: - constraints: - description: Optional. - items: - properties: - key: - description: Key of the constraint. - format: string - type: string - values: - description: List of valid values for the constraint. - items: - format: string - type: string - type: array - type: object - type: array - hosts: - items: - format: string - type: string - type: array - methods: - description: Optional. - items: - format: string - type: string - type: array - notHosts: - items: - format: string - type: string - type: array - notMethods: - items: - format: string - type: string - type: array - notPaths: - items: - format: string - type: string - type: array - notPorts: - items: - format: int32 - type: integer - type: array - paths: - description: Optional. - items: - format: string - type: string - type: array - ports: - items: - format: int32 - type: integer - type: array - services: - description: A list of service names. - items: - format: string - type: string - type: array - type: object - type: array - mode: - enum: - - ENFORCED - - PERMISSIVE - type: string - role: - format: string - type: string - roleRef: - description: Reference to the ServiceRole object. - properties: - kind: - description: The type of the role being referenced. - format: string - type: string - name: - description: The name of the ServiceRole object being referenced. - format: string - type: string - type: object - subjects: - description: List of subjects that are assigned the ServiceRole object. - items: - properties: - group: - format: string - type: string - groups: - items: - format: string - type: string - type: array - ips: - items: - format: string - type: string - type: array - names: - items: - format: string - type: string - type: array - namespaces: - items: - format: string - type: string - type: array - notGroups: - items: - format: string - type: string - type: array - notIps: - items: - format: string - type: string - type: array - notNames: - items: - format: string - type: string - type: array - notNamespaces: - items: - format: string - type: string - type: array - properties: - additionalProperties: - format: string - type: string - description: Optional. - type: object - user: - description: Optional. - format: string - type: string - type: object - type: array - type: object - type: object - versions: - - name: v1alpha1 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: mixer - chart: istio - heritage: Tiller - istio: rbac - package: istio.io.mixer - release: istio - name: serviceroles.rbac.istio.io -spec: - group: rbac.istio.io - names: - categories: - - istio-io - - rbac-istio-io - kind: ServiceRole - plural: serviceroles - singular: servicerole - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration for Role Based Access Control. See more details - at: https://istio.io/docs/reference/config/authorization/istio.rbac.v1alpha1.html' - properties: - rules: - description: The set of access rules (permissions) that the role has. - items: - properties: - constraints: - description: Optional. - items: - properties: - key: - description: Key of the constraint. - format: string - type: string - values: - description: List of valid values for the constraint. - items: - format: string - type: string - type: array - type: object - type: array - hosts: - items: - format: string - type: string - type: array - methods: - description: Optional. - items: - format: string - type: string - type: array - notHosts: - items: - format: string - type: string - type: array - notMethods: - items: - format: string - type: string - type: array - notPaths: - items: - format: string - type: string - type: array - notPorts: - items: - format: int32 - type: integer - type: array - paths: - description: Optional. - items: - format: string - type: string - type: array - ports: - items: - format: int32 - type: integer - type: array - services: - description: A list of service names. - items: - format: string - type: string - type: array - type: object - type: array - type: object - type: object - versions: - - name: v1alpha1 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: virtualservices.networking.istio.io -spec: - additionalPrinterColumns: - - JSONPath: .spec.gateways - description: The names of gateways and sidecars that should apply these routes - name: Gateways - type: string - - JSONPath: .spec.hosts - description: The destination hosts to which traffic is being sent - name: Hosts - type: string - - JSONPath: .metadata.creationTimestamp - description: |- - CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC. - Populated by the system. Read-only. Null for lists. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata - name: Age - type: date - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: VirtualService - listKind: VirtualServiceList - plural: virtualservices - shortNames: - - vs - singular: virtualservice - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting label/content routing, sni routing, - etc. See more details at: https://istio.io/docs/reference/config/networking/v1alpha3/virtual-service.html' - properties: - exportTo: - description: A list of namespaces to which this virtual service is exported. - items: - format: string - type: string - type: array - gateways: - description: The names of gateways and sidecars that should apply these - routes. - items: - format: string - type: string - type: array - hosts: - description: The destination hosts to which traffic is being sent. - items: - format: string - type: string - type: array - http: - description: An ordered list of route rules for HTTP traffic. - items: - properties: - appendHeaders: - additionalProperties: - format: string - type: string - type: object - appendRequestHeaders: - additionalProperties: - format: string - type: string - type: object - appendResponseHeaders: - additionalProperties: - format: string - type: string - type: object - corsPolicy: - description: Cross-Origin Resource Sharing policy (CORS). - properties: - allowCredentials: - nullable: true - type: boolean - allowHeaders: - items: - format: string - type: string - type: array - allowMethods: - description: List of HTTP methods allowed to access the resource. - items: - format: string - type: string - type: array - allowOrigin: - description: The list of origins that are allowed to perform - CORS requests. - items: - format: string - type: string - type: array - exposeHeaders: - items: - format: string - type: string - type: array - maxAge: - type: string - type: object - fault: - description: Fault injection policy to apply on HTTP traffic at - the client side. - properties: - abort: - oneOf: - - properties: - percent: {} - required: - - httpStatus - - properties: - percent: {} - required: - - grpcStatus - - properties: - percent: {} - required: - - http2Error - properties: - grpcStatus: - format: string - type: string - http2Error: - format: string - type: string - httpStatus: - description: HTTP status code to use to abort the Http - request. - format: int32 - type: integer - percent: - description: Percentage of requests to be aborted with - the error code provided (0-100). - format: int32 - type: integer - percentage: - description: Percentage of requests to be aborted with - the error code provided. - properties: - value: - format: double - type: number - type: object - type: object - delay: - oneOf: - - properties: - percent: {} - required: - - fixedDelay - - properties: - percent: {} - required: - - exponentialDelay - properties: - exponentialDelay: - type: string - fixedDelay: - description: Add a fixed delay before forwarding the request. - type: string - percent: - description: Percentage of requests on which the delay - will be injected (0-100). - format: int32 - type: integer - percentage: - description: Percentage of requests on which the delay - will be injected. - properties: - value: - format: double - type: number - type: object - type: object - type: object - headers: - properties: - request: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - response: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - match: - items: - properties: - authority: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - format: string - type: string - type: object - gateways: - items: - format: string - type: string - type: array - headers: - additionalProperties: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - format: string - type: string - type: object - type: object - ignoreUriCase: - description: Flag to specify whether the URI matching should - be case-insensitive. - type: boolean - method: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - format: string - type: string - type: object - name: - description: The name assigned to a match. - format: string - type: string - port: - description: Specifies the ports on the host that is being - addressed. - type: integer - queryParams: - additionalProperties: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - format: string - type: string - type: object - description: Query parameters for matching. - type: object - scheme: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - format: string - type: string - type: object - sourceLabels: - additionalProperties: - format: string - type: string - type: object - uri: - oneOf: - - required: - - exact - - required: - - prefix - - required: - - regex - properties: - exact: - format: string - type: string - prefix: - format: string - type: string - regex: - format: string - type: string - type: object - type: object - type: array - mirror: - properties: - host: - description: The name of a service from the service registry. - format: string - type: string - port: - description: Specifies the port on the host that is being - addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - mirror_percent: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - nullable: true - type: integer - mirrorPercent: - description: Percentage of the traffic to be mirrored by the `mirror` - field. - nullable: true - type: integer - name: - description: The name assigned to the route for debugging purposes. - format: string - type: string - redirect: - description: A http rule can either redirect or forward (default) - traffic. - properties: - authority: - format: string - type: string - redirectCode: - type: integer - uri: - format: string - type: string - type: object - removeRequestHeaders: - items: - format: string - type: string - type: array - removeResponseHeaders: - items: - format: string - type: string - type: array - retries: - description: Retry policy for HTTP requests. - properties: - attempts: - description: Number of retries for a given request. - format: int32 - type: integer - perTryTimeout: - description: Timeout per retry attempt for a given request. - type: string - retryOn: - description: Specifies the conditions under which retry takes - place. - format: string - type: string - type: object - rewrite: - description: Rewrite HTTP URIs and Authority headers. - properties: - authority: - description: rewrite the Authority/Host header with this value. - format: string - type: string - uri: - format: string - type: string - type: object - route: - description: A http rule can either redirect or forward (default) - traffic. - items: - properties: - appendRequestHeaders: - additionalProperties: - format: string - type: string - description: Use of `append_request_headers` is deprecated. - type: object - appendResponseHeaders: - additionalProperties: - format: string - type: string - description: Use of `append_response_headers` is deprecated. - type: object - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - headers: - properties: - request: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - response: - properties: - add: - additionalProperties: - format: string - type: string - type: object - remove: - items: - format: string - type: string - type: array - set: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - removeRequestHeaders: - description: Use of `remove_request_headers` is deprecated. - items: - format: string - type: string - type: array - removeResponseHeaders: - description: Use of `remove_response_header` is deprecated. - items: - format: string - type: string - type: array - weight: - format: int32 - type: integer - type: object - type: array - timeout: - description: Timeout for HTTP requests. - type: string - websocketUpgrade: - description: Deprecated. - type: boolean - type: object - type: array - tcp: - description: An ordered list of route rules for opaque TCP traffic. - items: - properties: - match: - items: - properties: - destinationSubnets: - description: IPv4 or IPv6 ip addresses of destination with - optional subnet. - items: - format: string - type: string - type: array - gateways: - description: Names of gateways where the rule should be - applied to. - items: - format: string - type: string - type: array - port: - description: Specifies the port on the host that is being - addressed. - type: integer - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceSubnet: - description: IPv4 or IPv6 ip address of source with optional - subnet. - format: string - type: string - type: object - type: array - route: - description: The destination to which the connection should be - forwarded to. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - weight: - format: int32 - type: integer - type: object - type: array - type: object - type: array - tls: - items: - properties: - match: - items: - properties: - destinationSubnets: - description: IPv4 or IPv6 ip addresses of destination with - optional subnet. - items: - format: string - type: string - type: array - gateways: - description: Names of gateways where the rule should be - applied to. - items: - format: string - type: string - type: array - port: - description: Specifies the port on the host that is being - addressed. - type: integer - sniHosts: - description: SNI (server name indicator) to match on. - items: - format: string - type: string - type: array - sourceLabels: - additionalProperties: - format: string - type: string - type: object - sourceSubnet: - description: IPv4 or IPv6 ip address of source with optional - subnet. - format: string - type: string - type: object - type: array - route: - description: The destination to which the connection should be - forwarded to. - items: - properties: - destination: - properties: - host: - description: The name of a service from the service - registry. - format: string - type: string - port: - description: Specifies the port on the host that is - being addressed. - properties: - number: - type: integer - type: object - subset: - description: The name of a subset within the service. - format: string - type: string - type: object - weight: - format: int32 - type: integer - type: object - type: array - type: object - type: array - type: object - type: object - versions: - - name: v1alpha3 - served: true - storage: true ---- - - -kind: CustomResourceDefinition -apiVersion: apiextensions.k8s.io/v1beta1 -metadata: - name: adapters.config.istio.io - labels: - app: mixer - package: adapter - istio: mixer-adapter - chart: istio - heritage: Tiller - release: istio -spec: - group: config.istio.io - names: - kind: adapter - plural: adapters - singular: adapter - categories: - - istio-io - - policy-istio-io - scope: Namespaced - subresources: - status: {} - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -kind: CustomResourceDefinition -apiVersion: apiextensions.k8s.io/v1beta1 -metadata: - name: instances.config.istio.io - labels: - app: mixer - package: instance - istio: mixer-instance - chart: istio - heritage: Tiller - release: istio -spec: - group: config.istio.io - names: - kind: instance - plural: instances - singular: instance - categories: - - istio-io - - policy-istio-io - scope: Namespaced - subresources: - status: {} - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -kind: CustomResourceDefinition -apiVersion: apiextensions.k8s.io/v1beta1 -metadata: - name: templates.config.istio.io - labels: - app: mixer - package: template - istio: mixer-template - chart: istio - heritage: Tiller - release: istio -spec: - group: config.istio.io - names: - kind: template - plural: templates - singular: template - categories: - - istio-io - - policy-istio-io - scope: Namespaced - subresources: - status: {} - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -kind: CustomResourceDefinition -apiVersion: apiextensions.k8s.io/v1beta1 -metadata: - name: handlers.config.istio.io - labels: - app: mixer - package: handler - istio: mixer-handler - chart: istio - heritage: Tiller - release: istio -spec: - group: config.istio.io - names: - kind: handler - plural: handlers - singular: handler - categories: - - istio-io - - policy-istio-io - scope: Namespaced - subresources: - status: {} - versions: - - name: v1alpha2 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - chart: istio - heritage: Tiller - release: istio - name: sidecars.networking.istio.io -spec: - group: networking.istio.io - names: - categories: - - istio-io - - networking-istio-io - kind: Sidecar - plural: sidecars - singular: sidecar - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration affecting network reachability of a sidecar. - See more details at: https://istio.io/docs/reference/config/networking/v1alpha3/sidecar.html' - properties: - egress: - items: - properties: - bind: - format: string - type: string - captureMode: - enum: - - DEFAULT - - IPTABLES - - NONE - type: string - hosts: - items: - format: string - type: string - type: array - port: - description: The port associated with the listener. - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - type: object - type: object - type: array - ingress: - items: - properties: - bind: - description: The ip to which the listener should be bound. - format: string - type: string - captureMode: - enum: - - DEFAULT - - IPTABLES - - NONE - type: string - defaultEndpoint: - format: string - type: string - port: - description: The port associated with the listener. - properties: - name: - description: Label assigned to the port. - format: string - type: string - number: - description: A valid non-negative integer port number. - type: integer - protocol: - description: The protocol exposed on the port. - format: string - type: string - type: object - type: object - type: array - outboundTrafficPolicy: - description: This allows to configure the outbound traffic policy. - properties: - mode: - enum: - - REGISTRY_ONLY - - ALLOW_ANY - type: string - type: object - workloadSelector: - properties: - labels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - type: object - versions: - - name: v1alpha3 - served: true - storage: true ---- - - -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - app: istio-pilot - heritage: Tiller - istio: security - release: istio - name: authorizationpolicies.security.istio.io -spec: - group: security.istio.io - names: - categories: - - istio-io - - security-istio-io - kind: AuthorizationPolicy - plural: authorizationpolicies - singular: authorizationpolicy - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - description: 'Configuration for access control on workloads. See more details - at: https://istio.io/docs/reference/config/security/v1beta1/authorization-policy.html' - properties: - rules: - description: Optional. - items: - properties: - from: - description: Optional. - items: - properties: - source: - description: Source specifies the source of a request. - properties: - ipBlocks: - description: Optional. - items: - format: string - type: string - type: array - namespaces: - description: Optional. - items: - format: string - type: string - type: array - principals: - description: Optional. - items: - format: string - type: string - type: array - requestPrincipals: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: object - type: array - to: - description: Optional. - items: - properties: - operation: - description: Operation specifies the operation of a request. - properties: - hosts: - description: Optional. - items: - format: string - type: string - type: array - methods: - description: Optional. - items: - format: string - type: string - type: array - paths: - description: Optional. - items: - format: string - type: string - type: array - ports: - description: Optional. - items: - format: string - type: string - type: array - type: object - type: object - type: array - when: - description: Optional. - items: - properties: - key: - description: The name of an Istio attribute. - format: string - type: string - values: - description: The allowed values for the attribute. - items: - format: string - type: string - type: array - type: object - type: array - type: object - type: array - selector: - description: Optional. - properties: - matchLabels: - additionalProperties: - format: string - type: string - type: object - type: object - type: object - type: object - versions: - - name: v1beta1 - served: true - storage: true ---- - - -apiVersion: v1 -kind: Namespace -metadata: - name: istio-system - labels: - istio-operator-managed: Reconcile - istio-injection: disabled ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-reader-service-account - namespace: istio-system - labels: - app: istio-reader - release: istio ---- - -# CertManager component is disabled. - -# Resources for Citadel component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-citadel-istio-system - labels: - app: citadel - release: istio -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "get", "update"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "get", "watch", "list", "update", "delete"] -- apiGroups: [""] - resources: ["serviceaccounts", "services", "namespaces"] - verbs: ["get", "watch", "list"] -- apiGroups: ["authentication.k8s.io"] - resources: ["tokenreviews"] - verbs: ["create"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: istio-citadel-istio-system - labels: - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: istio-citadel-istio-system -subjects: - - kind: ServiceAccount - name: istio-citadel-service-account - namespace: istio-system ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: security - istio: citadel - release: istio - name: istio-citadel - namespace: istio-system -spec: - replicas: 1 - selector: - matchLabels: - istio: citadel - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: citadel - istio: citadel - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - args: - - --append-dns-names=true - - --grpc-port=8060 - - --citadel-storage-namespace=istio-system - - --custom-dns-names=istio-galley-service-account.istio-config:istio-galley.istio-config.svc,istio-galley-service-account.istio-control:istio-galley.istio-control.svc,istio-galley-service-account.istio-control-master:istio-galley.istio-control-master.svc,istio-galley-service-account.istio-master:istio-galley.istio-master.svc,istio-galley-service-account.istio-pilot11:istio-galley.istio-pilot11.svc,istio-pilot-service-account.istio-control:istio-pilot.istio-control,istio-pilot-service-account.istio-pilot11:istio-pilot.istio-system,istio-sidecar-injector-service-account.istio-control:istio-sidecar-injector.istio-control.svc,istio-sidecar-injector-service-account.istio-control-master:istio-sidecar-injector.istio-control-master.svc,istio-sidecar-injector-service-account.istio-master:istio-sidecar-injector.istio-master.svc,istio-sidecar-injector-service-account.istio-pilot11:istio-sidecar-injector.istio-pilot11.svc,istio-sidecar-injector-service-account.istio-remote:istio-sidecar-injector.istio-remote.svc, - - --self-signed-ca=true - - --trust-domain=cluster.local - - --workload-cert-ttl=2160h - env: - - name: CITADEL_ENABLE_NAMESPACES_BY_DEFAULT - value: "true" - image: docker.io/istio/citadel:1.4.2 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /version - port: 15014 - initialDelaySeconds: 5 - periodSeconds: 5 - name: citadel - resources: - requests: - cpu: 10m - serviceAccountName: istio-citadel-service-account - ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: istio-citadel - namespace: istio-system - labels: - app: security - istio: citadel - release: istio -spec: - minAvailable: 1 - selector: - matchLabels: - app: citadel - istio: citadel ---- - - -apiVersion: v1 -kind: Service -metadata: - # Must match the certificate, this is used in the node agent in same namespace. - name: istio-citadel - namespace: istio-system - labels: - app: security - istio: citadel - release: istio - -spec: - ports: - - name: grpc-citadel - port: 8060 - targetPort: 8060 - protocol: TCP - - name: http-monitoring - port: 15014 - selector: - app: citadel ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-citadel-service-account - namespace: istio-system - labels: - app: security - release: istio ---- - -# Cni component is disabled. - -# CoreDNS component is disabled. - -# Resources for EgressGateway component - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: istio-egressgateway - istio: egressgateway - release: istio - name: istio-egressgateway - namespace: istio-system -spec: - selector: - matchLabels: - app: istio-egressgateway - istio: egressgateway - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: istio-egressgateway - chart: gateways - heritage: Tiller - istio: egressgateway - release: istio - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - args: - - proxy - - router - - --domain - - $(POD_NAMESPACE).svc.cluster.local - - --proxyLogLevel=warning - - --proxyComponentLogLevel=misc:error - - --log_output_level=default:info - - --drainDuration - - 45s - - --parentShutdownDuration - - 1m0s - - --connectTimeout - - 10s - - --serviceCluster - - istio-egressgateway - - --zipkinAddress - - zipkin.istio-system:9411 - - --proxyAdminPort - - "15000" - - --statusPort - - "15020" - - --controlPlaneAuthPolicy - - NONE - - --discoveryAddress - - istio-pilot.istio-system:15010 - - --trust-domain=cluster.local - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: INSTANCE_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - name: SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: ISTIO_META_WORKLOAD_NAME - value: istio-egressgateway - - name: ISTIO_META_OWNER - value: kubernetes://api/apps/v1/namespaces/istio-system/deployments/istio-egressgateway - - name: ISTIO_META_MESH_ID - value: cluster.local - - name: ISTIO_META_POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: ISTIO_META_CONFIG_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ISTIO_META_ROUTER_MODE - value: sni-dnat - - name: ISTIO_METAJSON_LABELS - value: | - {"app":"istio-egressgateway","istio":"egressgateway"} - - name: ISTIO_META_CLUSTER_ID - value: Kubernetes - - name: SDS_ENABLED - value: "false" - image: docker.io/istio/proxyv2:1.4.2 - imagePullPolicy: IfNotPresent - name: istio-proxy - ports: - - containerPort: 80 - - containerPort: 443 - - containerPort: 15443 - - containerPort: 15090 - name: http-envoy-prom - protocol: TCP - readinessProbe: - failureThreshold: 30 - httpGet: - path: /healthz/ready - port: 15020 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 2 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - cpu: 2000m - memory: 1024Mi - requests: - cpu: 10m - memory: 40Mi - volumeMounts: - - mountPath: /etc/certs - name: istio-certs - readOnly: true - - mountPath: /etc/istio/egressgateway-certs - name: egressgateway-certs - readOnly: true - - mountPath: /etc/istio/egressgateway-ca-certs - name: egressgateway-ca-certs - readOnly: true - volumes: - - name: istio-certs - secret: - optional: true - secretName: istio.default - - name: egressgateway-certs - secret: - optional: true - secretName: istio-egressgateway-certs - - name: egressgateway-ca-certs - secret: - optional: true - secretName: istio-egressgateway-ca-certs - ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: istio-egressgateway - namespace: istio-system - labels: - app: istio-egressgateway - istio: egressgateway - release: istio -spec: - minAvailable: 1 - selector: - matchLabels: - app: istio-egressgateway - istio: egressgateway - release: istio ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: istio-multicluster-egressgateway - namespace: istio-system - labels: - app: istio-egressgateway - release: istio -spec: - selector: - istio: egressgateway - servers: - - hosts: - - "*.global" - port: - name: tls - number: 15443 - protocol: TLS - tls: {} ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: VirtualService -metadata: - name: istio-multicluster-egressgateway - namespace: istio-system - labels: - app: istio-egressgateway - release: istio -spec: - gateways: - - istio-multicluster-egressgateway - hosts: - - "*.global" - tls: - - match: - - port: 15443 - sniHosts: - - "*.global" - route: - - destination: - host: non.existent.cluster - port: - number: 15443 - weight: 100 ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: EnvoyFilter -metadata: - name: istio-multicluster-egressgateway - namespace: istio-system - labels: - app: istio-egressgateway - release: istio -spec: - workloadLabels: - istio: egressgateway - filters: - - listenerMatch: - portNumber: 15443 - listenerType: GATEWAY - filterName: envoy.filters.network.sni_cluster - filterType: NETWORK - filterConfig: {} ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: istio-multicluster-destinationrule - namespace: istio-system - labels: - app: istio-egressgateway - release: istio -spec: - host: "*.global" - trafficPolicy: - tls: - mode: ISTIO_MUTUAL ---- - - -apiVersion: v1 -kind: Service -metadata: - name: istio-egressgateway - namespace: istio-system - annotations: - labels: - app: istio-egressgateway - release: istio - istio: egressgateway -spec: - type: ClusterIP - selector: - app: istio-egressgateway - ports: - - - name: http2 - port: 80 - - - name: https - port: 443 - - - name: tls - port: 15443 - targetPort: 15443 ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: istio-egressgateway-sds - namespace: istio-system -rules: -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "watch", "list"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: istio-egressgateway-sds - namespace: istio-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: istio-egressgateway-sds -subjects: -- kind: ServiceAccount - name: istio-egressgateway-service-account ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-egressgateway-service-account - namespace: istio-system - labels: - app: istio-egressgateway - release: istio ---- - -# Resources for Galley component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-galley-istio-system - labels: - release: istio -rules: - # For reading Istio resources - - apiGroups: [ - "authentication.istio.io", - "config.istio.io", - "networking.istio.io", - "rbac.istio.io", - "security.istio.io"] - resources: ["*"] - verbs: ["get", "list", "watch"] - # For updating Istio resource statuses - - apiGroups: [ - "authentication.istio.io", - "config.istio.io", - "networking.istio.io", - "rbac.istio.io", - "security.istio.io"] - resources: ["*/status"] - verbs: ["update"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["*"] - - apiGroups: ["extensions","apps"] - resources: ["deployments"] - resourceNames: ["istio-galley"] - verbs: ["get"] - - apiGroups: [""] - resources: ["pods", "nodes", "services", "endpoints", "namespaces"] - verbs: ["get", "list", "watch"] - - apiGroups: ["extensions"] - resources: ["ingresses"] - verbs: ["get", "list", "watch"] - - apiGroups: ["extensions"] - resources: ["deployments/finalizers"] - resourceNames: ["istio-galley"] - verbs: ["update"] - - apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "watch"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: istio-galley-admin-role-binding-istio-system - labels: - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: istio-galley-istio-system -subjects: - - kind: ServiceAccount - name: istio-galley-service-account - namespace: istio-system ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-mesh-galley - namespace: istio-system - labels: - release: istio -data: - mesh: |- - {} ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-galley-configuration - namespace: istio-system - labels: - release: istio -data: - validatingwebhookconfiguration.yaml: |- - apiVersion: admissionregistration.k8s.io/v1beta1 - kind: ValidatingWebhookConfiguration - metadata: - name: istio-galley-istio-system - namespace: istio-system - labels: - app: galley - release: istio - istio: galley - webhooks: - - name: pilot.validation.istio.io - clientConfig: - service: - name: istio-galley - namespace: istio-system - path: "/admitpilot" - caBundle: "" - rules: - - operations: - - CREATE - - UPDATE - apiGroups: - - config.istio.io - apiVersions: - - v1alpha2 - resources: - - httpapispecs - - httpapispecbindings - - quotaspecs - - quotaspecbindings - - operations: - - CREATE - - UPDATE - apiGroups: - - rbac.istio.io - apiVersions: - - "*" - resources: - - "*" - - operations: - - CREATE - - UPDATE - apiGroups: - - security.istio.io - apiVersions: - - "*" - resources: - - "*" - - operations: - - CREATE - - UPDATE - apiGroups: - - authentication.istio.io - apiVersions: - - "*" - resources: - - "*" - - operations: - - CREATE - - UPDATE - apiGroups: - - networking.istio.io - apiVersions: - - "*" - resources: - - destinationrules - - envoyfilters - - gateways - - serviceentries - - sidecars - - virtualservices - failurePolicy: Fail - sideEffects: None - - name: mixer.validation.istio.io - clientConfig: - service: - name: istio-galley - namespace: istio-system - path: "/admitmixer" - caBundle: "" - rules: - - operations: - - CREATE - - UPDATE - apiGroups: - - config.istio.io - apiVersions: - - v1alpha2 - resources: - - rules - - attributemanifests - - circonuses - - deniers - - fluentds - - kubernetesenvs - - listcheckers - - memquotas - - noops - - opas - - prometheuses - - rbacs - - solarwindses - - stackdrivers - - cloudwatches - - dogstatsds - - statsds - - stdios - - apikeys - - authorizations - - checknothings - # - kuberneteses - - listentries - - logentries - - metrics - - quotas - - reportnothings - - tracespans - - adapters - - handlers - - instances - - templates - - zipkins - failurePolicy: Fail - sideEffects: None ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: galley - istio: galley - release: istio - name: istio-galley - namespace: istio-system -spec: - replicas: 1 - selector: - matchLabels: - istio: galley - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: galley - chart: galley - heritage: Tiller - istio: galley - release: istio - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - command: - - /usr/local/bin/galley - - server - - --meshConfigFile=/etc/mesh-config/mesh - - --livenessProbeInterval=1s - - --livenessProbePath=/tmp/healthliveness - - --readinessProbePath=/tmp/healthready - - --readinessProbeInterval=1s - - --insecure=true - - --enable-validation=true - - --enable-reconcileWebhookConfiguration=true - - --enable-server=true - - --deployment-namespace=istio-system - - --validation-webhook-config-file - - /etc/config/validatingwebhookconfiguration.yaml - - --monitoringPort=15014 - - --validation-port=9443 - - --log_output_level=default:info - image: docker.io/istio/galley:1.4.2 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - /usr/local/bin/galley - - probe - - --probe-path=/tmp/healthliveness - - --interval=10s - initialDelaySeconds: 5 - periodSeconds: 5 - name: galley - ports: - - containerPort: 9443 - - containerPort: 15014 - - containerPort: 15019 - - containerPort: 9901 - readinessProbe: - exec: - command: - - /usr/local/bin/galley - - probe - - --probe-path=/tmp/healthready - - --interval=10s - initialDelaySeconds: 5 - periodSeconds: 5 - resources: - requests: - cpu: 100m - volumeMounts: - - mountPath: /etc/certs - name: istio-certs - readOnly: true - - mountPath: /etc/config - name: config - readOnly: true - - mountPath: /etc/mesh-config - name: mesh-config - readOnly: true - serviceAccountName: istio-galley-service-account - volumes: - - name: istio-certs - secret: - secretName: istio.istio-galley-service-account - - configMap: - name: istio-galley-configuration - name: config - - configMap: - name: istio-mesh-galley - name: mesh-config - ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: istio-galley - namespace: istio-system - labels: - app: galley - release: istio - istio: galley -spec: - minAvailable: 1 - selector: - matchLabels: - app: galley - release: istio - istio: galley ---- - - -apiVersion: v1 -kind: Service -metadata: - name: istio-galley - namespace: istio-system - labels: - app: galley - istio: galley - release: istio -spec: - ports: - - port: 443 - name: https-validation - targetPort: 9443 - - port: 15014 - name: http-monitoring - - port: 9901 - name: grpc-mcp - - port: 15019 - name: grpc-tls-mcp - selector: - istio: galley ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-galley-service-account - namespace: istio-system - labels: - app: galley - release: istio ---- - -# Resources for Grafana component - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-citadel-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - citadel-dashboard.json: '{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "", - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 8, - "panels": [], - "title": "Performance", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "CPU usage across Citadel instances.", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 10, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"citadel\", pod_name=~\"istio-citadel-.*\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Citadel CPU usage rate", - "refId": "A" - }, - { - "expr": "irate(process_cpu_seconds_total{job=\"citadel\"}[1m])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Citadel CPU usage irate", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Citadel process memory statistics.", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_virtual_memory_bytes{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Virtual Memory", - "refId": "A" - }, - { - "expr": "process_resident_memory_bytes{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Resident Memory", - "refId": "B" - }, - { - "expr": "go_memstats_heap_sys_bytes{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Heap Memory Total", - "refId": "C" - }, - { - "expr": "go_memstats_alloc_bytes{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Heap Memory Allocated", - "refId": "E" - }, - { - "expr": "go_memstats_heap_inuse_bytes{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Heap Inuse", - "refId": "F" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Goroutines", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 7 - }, - "id": 28, - "panels": [], - "title": "General", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Total number of CSR requests made to Citadel.", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 8 - }, - "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "citadel_server_csr_count{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CSR Request Count", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CSR Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The number of certificates issuances that have succeeded.", - "fill": 1, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 8 - }, - "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "citadel_server_success_cert_issuance_count{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Certificates Issued", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Certificates Issued", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 13 - }, - "id": 23, - "panels": [], - "title": "Errors", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The number of errors occurred when creating the CSR.", - "fill": 1, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 14 - }, - "id": 20, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "citadel_secret_controller_csr_err_count{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CSR Creation Error Count", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CSR Creation Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 14 - }, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "citadel_server_csr_parsing_err_count{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "CSR Parse Error Count", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CSR Parse Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The number of authentication failures.", - "fill": 1, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 14 - }, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "citadel_server_authentication_failure_count{job=\"citadel\"}\t", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Authentication Failure Count", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Authentication Failures", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 4, - "panels": [], - "title": "Secret Controller", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The number of certificates created due to service account creation.", - "fill": 1, - "gridPos": { - "h": 5, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "citadel_secret_controller_svc_acc_created_cert_count{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SA Secrets Created", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Service Account Secrets Created (due to SA creation)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "Certs Created", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The number of certificates deleted due to service account deletion.", - "fill": 1, - "gridPos": { - "h": 5, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "citadel_secret_controller_svc_acc_deleted_cert_count{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SA Secrets Deleted", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Service Account Secrets Deleted (due to SA deletion)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "Certs Created", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "The number of certificates recreated due to secret deletion (service account still exists).", - "fill": 1, - "gridPos": { - "h": 5, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": true, - "targets": [ - { - "expr": "citadel_secret_controller_secret_deleted_cert_count{job=\"citadel\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "SA Secrets Recreated", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Service Account Secrets Recreated (due to errant deletion)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": "Certs Created", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 18, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Istio Citadel Dashboard", - "uid": "OOyOqb4Wz", - "version": 1 -}' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-galley-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - galley-dashboard.json: '{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(istio_build{component=\"galley\"}) by (tag)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ tag }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Galley Versions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 40, - "panels": [], - "title": "Resource Usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 6 - }, - "id": 36, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_virtual_memory_bytes{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Virtual Memory", - "refId": "A" - }, - { - "expr": "process_resident_memory_bytes{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Resident Memory", - "refId": "B" - }, - { - "expr": "go_memstats_heap_sys_bytes{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "heap sys", - "refId": "C" - }, - { - "expr": "go_memstats_heap_alloc_bytes{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "heap alloc", - "refId": "D" - }, - { - "expr": "go_memstats_alloc_bytes{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Alloc", - "refId": "F" - }, - { - "expr": "go_memstats_heap_inuse_bytes{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Heap in-use", - "refId": "G" - }, - { - "expr": "go_memstats_stack_inuse_bytes{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Stack in-use", - "refId": "H" - }, - { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Total (kis)", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 6 - }, - "id": 38, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"}[1m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Total (k8s)", - "refId": "A" - }, - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"}[1m])) by (container_name)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_name }} (k8s)", - "refId": "B" - }, - { - "expr": "irate(process_cpu_seconds_total{job=\"galley\"}[1m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "galley (self-reported)", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 6 - }, - "id": 42, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Open FDs (galley)", - "refId": "A" - }, - { - "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"galley\", pod_name=~\"istio-galley-.*\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_name }} ", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 6 - }, - "id": 44, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "goroutines_total", - "refId": "A" - }, - { - "expr": "galley_mcp_source_clients_total", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "clients_total", - "refId": "B" - }, - { - "expr": "go_goroutines{job=\"galley\"}/galley_mcp_source_clients_total", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "avg_goroutines_per_client", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 10, - "panels": [], - "title": "Runtime", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(galley_runtime_strategy_on_change_total[1m])) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Strategy Change Events", - "refId": "A" - }, - { - "expr": "sum(rate(galley_runtime_processor_events_processed_total[1m])) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Processed Events", - "refId": "B" - }, - { - "expr": "sum(rate(galley_runtime_processor_snapshots_published_total[1m])) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Snapshot Published", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Event Rates", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Events/min", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(galley_runtime_strategy_timer_max_time_reached_total[1m])) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Max Time Reached", - "refId": "A" - }, - { - "expr": "sum(rate(galley_runtime_strategy_timer_quiesce_reached_total[1m])) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Quiesce Reached", - "refId": "B" - }, - { - "expr": "sum(rate(galley_runtime_strategy_timer_resets_total[1m])) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Timer Resets", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Timer Rates", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Events/min", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 15 - }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 3, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "P50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.90, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "P90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.95, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "P95", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.99, sum by (le) (galley_runtime_processor_snapshot_events_total_bucket))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "P99", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Events Per Snapshot", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 21 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (collection) (galley_runtime_state_type_instances_total)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ collection }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "State Type Instances", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Count", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 34, - "panels": [], - "title": "Validation", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 28 - }, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "galley_validation_cert_key_updates{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Key Updates", - "refId": "A" - }, - { - "expr": "galley_validation_cert_key_update_errors{job=\"galley\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Key Update Errors: {{ error }}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Validation Webhook Certificate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 28 - }, - "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(galley_validation_passed{job=\"galley\"}) by (group, version, resource)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Passed: {{ group }}/{{ version }}/{{resource}}", - "refId": "A" - }, - { - "expr": "sum(galley_validation_failed{job=\"galley\"}) by (group, version, resource, reason)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Failed: {{ group }}/{{ version }}/{{resource}} ({{ reason}})", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Resource Validation", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 28 - }, - "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(galley_validation_http_error{job=\"galley\"}) by (status)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ status }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Validation HTTP Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 12, - "panels": [], - "title": "Kubernetes Source", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 35 - }, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(galley_source_kube_event_success_total[1m]) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Success", - "refId": "A" - }, - { - "expr": "rate(galley_source_kube_event_error_total[1m]) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Error", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Source Event Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Events/min", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 35 - }, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(galley_source_kube_dynamic_converter_failure_total[1m]) * 60", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Error", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Kubernetes Object Conversion Failures", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Failures/min", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 41 - }, - "id": 18, - "panels": [], - "title": "Mesh Configuration Protocol", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 42 - }, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(galley_mcp_source_clients_total)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Clients", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Connected Clients", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 42 - }, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(collection)(irate(galley_mcp_source_request_acks_total[1m]) * 60)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request ACKs", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "ACKs/min", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 42 - }, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(galley_mcp_source_request_nacks_total[1m]) * 60", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Request NACKs", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "NACKs/min", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Istio Galley Dashboard", - "uid": "TSEY6jLmk", - "version": 1 -} -' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-istio-mesh-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - istio-mesh-dashboard.json: '{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.2.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "text", - "name": "Text", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "links": [], - "panels": [ - { - "content": "
\n
\n Istio\n
\n
\n Istio is an open platform that provides a uniform way to connect,\n manage, and \n secure microservices.\n
\n Need help? Join the Istio community.\n
\n
", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 0 - }, - "height": "50px", - "id": 13, - "links": [], - "mode": "html", - "style": { - "font-size": "18pt" - }, - "title": "", - "transparent": true, - "type": "text" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "Prometheus", - "format": "ops", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 0, - "y": 3 - }, - "id": 20, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\"}[1m])), 0.001)", - "intervalFactor": 1, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Global Request Volume", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "Prometheus", - "format": "percentunit", - "gauge": { - "maxValue": 100, - "minValue": 80, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 6, - "y": 3 - }, - "id": 21, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(rate(istio_requests_total{reporter=\"destination\", response_code!~\"5.*\"}[1m])) / sum(rate(istio_requests_total{reporter=\"destination\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "refId": "A", - "step": 4 - } - ], - "thresholds": "95, 99, 99.5", - "title": "Global Success Rate (non-5xx responses)", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "Prometheus", - "format": "ops", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 12, - "y": 3 - }, - "id": 22, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\", response_code=~\"4.*\"}[1m])) ", - "format": "time_series", - "intervalFactor": 1, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "4xxs", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "Prometheus", - "format": "ops", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 6, - "x": 18, - "y": 3 - }, - "id": 23, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\", response_code=~\"5.*\"}[1m])) ", - "format": "time_series", - "intervalFactor": 1, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "5xxs", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "columns": [], - "datasource": "Prometheus", - "fontSize": "100%", - "gridPos": { - "h": 21, - "w": 24, - "x": 0, - "y": 6 - }, - "hideTimeOverride": false, - "id": 73, - "links": [], - "pageSize": null, - "repeat": null, - "repeatDirection": "v", - "scroll": true, - "showHeader": true, - "sort": { - "col": 4, - "desc": true - }, - "styles": [ - { - "alias": "Workload", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "Workload dashboard", - "linkUrl": "/dashboard/db/istio-workload-dashboard?var-namespace=$__cell_2&var-workload=$__cell_", - "pattern": "destination_workload", - "preserveFormat": false, - "sanitize": false, - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Time", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Requests", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value #A", - "thresholds": [], - "type": "number", - "unit": "ops" - }, - { - "alias": "P50 Latency", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value #B", - "thresholds": [], - "type": "number", - "unit": "s" - }, - { - "alias": "P90 Latency", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value #D", - "thresholds": [], - "type": "number", - "unit": "s" - }, - { - "alias": "P99 Latency", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value #E", - "thresholds": [], - "type": "number", - "unit": "s" - }, - { - "alias": "Success Rate", - "colorMode": "cell", - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value #F", - "thresholds": [ - ".95", - " 1.00" - ], - "type": "number", - "unit": "percentunit" - }, - { - "alias": "Workload", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "$__cell dashboard", - "linkUrl": "/dashboard/db/istio-workload-dashboard?var-workload=$__cell_2&var-namespace=$__cell_3", - "pattern": "destination_workload_var", - "thresholds": [], - "type": "number", - "unit": "short" - }, - { - "alias": "Service", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "$__cell dashboard", - "linkUrl": "/dashboard/db/istio-service-dashboard?var-service=$__cell", - "pattern": "destination_service", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "destination_workload_namespace", - "thresholds": [], - "type": "hidden", - "unit": "short" - } - ], - "targets": [ - { - "expr": "label_join(sum(rate(istio_requests_total{reporter=\"destination\", response_code=\"200\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", - "format": "table", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload}}.{{ destination_workload_namespace }}", - "refId": "A" - }, - { - "expr": "label_join((histogram_quantile(0.50, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.50, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", - "format": "table", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload}}.{{ destination_workload_namespace }}", - "refId": "B" - }, - { - "expr": "label_join((histogram_quantile(0.90, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.90, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", - "format": "table", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", - "refId": "D" - }, - { - "expr": "label_join((histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)) / 1000) or histogram_quantile(0.99, sum(rate(istio_request_duration_seconds_bucket{reporter=\"destination\"}[1m])) by (le, destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", - "format": "table", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", - "refId": "E" - }, - { - "expr": "label_join((sum(rate(istio_requests_total{reporter=\"destination\", response_code!~\"5.*\"}[1m])) by (destination_workload, destination_workload_namespace) / sum(rate(istio_requests_total{reporter=\"destination\"}[1m])) by (destination_workload, destination_workload_namespace)), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", - "format": "table", - "hide": false, - "instant": true, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", - "refId": "F" - } - ], - "timeFrom": null, - "title": "HTTP/GRPC Workloads", - "transform": "table", - "transparent": false, - "type": "table" - }, - { - "columns": [], - "datasource": "Prometheus", - "fontSize": "100%", - "gridPos": { - "h": 18, - "w": 24, - "x": 0, - "y": 27 - }, - "hideTimeOverride": false, - "id": 109, - "links": [], - "pageSize": null, - "repeatDirection": "v", - "scroll": true, - "showHeader": true, - "sort": { - "col": 2, - "desc": true - }, - "styles": [ - { - "alias": "Workload", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": false, - "linkTargetBlank": false, - "linkTooltip": "$__cell dashboard", - "linkUrl": "/dashboard/db/istio-tcp-workload-dashboard?var-namespace=$__cell_2&&var-workload=$__cell", - "pattern": "destination_workload", - "preserveFormat": false, - "sanitize": false, - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Bytes Sent", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value #A", - "thresholds": [ - "" - ], - "type": "number", - "unit": "Bps" - }, - { - "alias": "Bytes Received", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Value #C", - "thresholds": [], - "type": "number", - "unit": "Bps" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "Time", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Workload", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "$__cell dashboard", - "linkUrl": "/dashboard/db/istio-workload-dashboard?var-namespace=$__cell_3&var-workload=$__cell_2", - "pattern": "destination_workload_var", - "thresholds": [], - "type": "string", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "pattern": "destination_workload_namespace", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "Service", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "link": true, - "linkTooltip": "$__cell dashboard", - "linkUrl": "/dashboard/db/istio-service-dashboard?var-service=$__cell", - "pattern": "destination_service", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "label_join(sum(rate(istio_tcp_received_bytes_total{reporter=\"source\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", - "format": "table", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}", - "refId": "C" - }, - { - "expr": "label_join(sum(rate(istio_tcp_sent_bytes_total{reporter=\"source\"}[1m])) by (destination_workload, destination_workload_namespace, destination_service), \"destination_workload_var\", \".\", \"destination_workload\", \"destination_workload_namespace\")", - "format": "table", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}", - "refId": "A" - } - ], - "timeFrom": null, - "title": "TCP Workloads", - "transform": "table", - "transparent": false, - "type": "table" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 45 - }, - "id": 111, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(istio_build) by (component, tag)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ component }}: {{ tag }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Istio Components by Version", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "transparent": false, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Istio Mesh Dashboard", - "version": 4 -} -' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-istio-performance-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - istio-performance-dashboard.json: '{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": 9, - "links": [], - "panels": [ - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 21, - "panels": [ - { - "content": "The charts on this dashboard are intended to show Istio main components cost in terms resources utilization under steady load.\n\n- **vCPU/1k rps:** shows vCPU utilization by the main Istio components normalized by 1000 requests/second. When idle or low traffic, this chart will be blank. The curve for istio-proxy refers to the services sidecars only.\n- **vCPU:** vCPU utilization by Istio components, not normalized.\n- **Memory:** memory footprint for the components. Telemetry and policy are normalized by 1k rps, and no data is shown when there is no traffic. For ingress and istio-proxy, the data is per instance.\n- **Bytes transferred/ sec:** shows the number of bytes flowing through each Istio component.\n\n\n", - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 19, - "links": [], - "mode": "markdown", - "timeFrom": null, - "timeShift": null, - "title": "Performance Dashboard README", - "transparent": true, - "type": "text" - } - ], - "title": "Performance Dashboard Notes", - "type": "row" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 6, - "panels": [], - "title": "vCPU Usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-telemetry-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))/ (round(sum(irate(istio_requests_total[1m])), 0.001)/1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "istio-telemetry", - "refId": "A" - }, - { - "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\",container_name=\"istio-proxy\"}[1m])) / (round(sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\", reporter=\"source\"}[1m])), 0.001)/1000))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "istio-ingressgateway", - "refId": "B" - }, - { - "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"}[1m]))/ (round(sum(irate(istio_requests_total[1m])), 0.001)/1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-proxy", - "refId": "C" - }, - { - "expr": "(sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-policy-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))/ (round(sum(irate(istio_requests_total[1m])), 0.001)/1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-policy", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "vCPU / 1k rps", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-telemetry-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-telemetry", - "refId": "A" - }, - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\",container_name=\"istio-proxy\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-ingressgateway", - "refId": "B" - }, - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-proxy", - "refId": "C" - }, - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",pod_name=~\"istio-policy-.*\",container_name=~\"mixer|istio-proxy\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-policy", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "vCPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 13, - "panels": [], - "title": "Memory and Data Rates", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 11 - }, - "id": 902, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-telemetry-.*\"}) / (sum(irate(istio_requests_total[1m])) / 1000)) / (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-telemetry / 1k rps", - "refId": "A" - }, - { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\"}) / count(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-ingressgateway-.*\",container_name!=\"POD\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "per istio-ingressgateway", - "refId": "B" - }, - { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"}) / count(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",namespace!=\"istio-system\",container_name=\"istio-proxy\"})", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "per istio proxy", - "refId": "C" - }, - { - "expr": "(sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",pod_name=~\"istio-policy-.*\"}) / (sum(irate(istio_requests_total[1m])) / 1000))/ (sum(irate(istio_requests_total{source_workload=\"istio-ingressgateway\"}[1m])) >bool 10)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-policy / 1k rps", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 11 - }, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(istio_response_bytes_sum{destination_workload=\"istio-telemetry\"}[1m])) + sum(irate(istio_request_bytes_sum{destination_workload=\"istio-telemetry\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-telemetry", - "refId": "A" - }, - { - "expr": "sum(irate(istio_response_bytes_sum{source_workload=\"istio-ingressgateway\", reporter=\"source\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-ingressgateway", - "refId": "B" - }, - { - "expr": "sum(irate(istio_response_bytes_sum{source_workload_namespace!=\"istio-system\", reporter=\"source\"}[1m])) + sum(irate(istio_response_bytes_sum{destination_workload_namespace!=\"istio-system\", reporter=\"destination\"}[1m])) + sum(irate(istio_request_bytes_sum{source_workload_namespace!=\"istio-system\", reporter=\"source\"}[1m])) + sum(irate(istio_request_bytes_sum{destination_workload_namespace!=\"istio-system\", reporter=\"destination\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio-proxy", - "refId": "C" - }, - { - "expr": "sum(irate(istio_response_bytes_sum{destination_workload=\"istio-policy\"}[1m])) + sum(irate(istio_request_bytes_sum{destination_workload=\"istio-policy\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "istio_policy", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Bytes transferred / sec", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 19 - }, - "id": 17, - "panels": [], - "title": "Istio Component Versions", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 20 - }, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(istio_build) by (component, tag)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ component }}: {{ tag }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Istio Components by Version", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 71, - "panels": [], - "title": "Proxy Resource Usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 32 - }, - "id": 72, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=\"istio-proxy\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ container_name }} (k8s)", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 32 - }, - "id": 73, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=\"istio-proxy\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Total (k8s)", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "vCPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 32 - }, - "id": 702, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=\"istio-proxy\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_name }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "none", - "label": "", - "logBase": 1024, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 69, - "panels": [], - "title": "Pilot Resource Usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 40 - }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_virtual_memory_bytes{job=\"pilot\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "Virtual Memory", - "refId": "I", - "step": 2 - }, - { - "expr": "process_resident_memory_bytes{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Resident Memory", - "refId": "H", - "step": 2 - }, - { - "expr": "go_memstats_heap_sys_bytes{job=\"pilot\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap sys", - "refId": "A" - }, - { - "expr": "go_memstats_heap_alloc_bytes{job=\"pilot\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap alloc", - "refId": "D" - }, - { - "expr": "go_memstats_alloc_bytes{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Alloc", - "refId": "F", - "step": 2 - }, - { - "expr": "go_memstats_heap_inuse_bytes{job=\"pilot\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Heap in-use", - "refId": "E", - "step": 2 - }, - { - "expr": "go_memstats_stack_inuse_bytes{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Stack in-use", - "refId": "G", - "step": 2 - }, - { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Total (k8s)", - "refId": "C", - "step": 2 - }, - { - "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ container_name }} (k8s)", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 40 - }, - "id": 602, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Total (k8s)", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}[1m])) by (container_name)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ container_name }} (k8s)", - "refId": "B", - "step": 2 - }, - { - "expr": "irate(process_cpu_seconds_total{job=\"pilot\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "pilot (self-reported)", - "refId": "C", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "vCPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 40 - }, - "id": 74, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{job=\"pilot\"}", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Open FDs (pilot)", - "refId": "A" - }, - { - "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"discovery|istio-proxy\", pod_name=~\"istio-pilot-.*\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_name }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "none", - "label": "", - "logBase": 1024, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 40 - }, - "id": 402, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Number of Goroutines", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 93, - "panels": [], - "title": "Mixer Resource Usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 48 - }, - "id": 94, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_virtual_memory_bytes{job=~\"istio-telemetry|istio-policy\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "Virtual Memory", - "refId": "I", - "step": 2 - }, - { - "expr": "process_resident_memory_bytes{job=~\"istio-telemetry|istio-policy\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Resident Memory", - "refId": "H", - "step": 2 - }, - { - "expr": "go_memstats_heap_sys_bytes{job=~\"istio-telemetry|istio-policy\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap sys", - "refId": "A" - }, - { - "expr": "go_memstats_heap_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap alloc", - "refId": "D" - }, - { - "expr": "go_memstats_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Alloc", - "refId": "F", - "step": 2 - }, - { - "expr": "go_memstats_heap_inuse_bytes{job=~\"istio-telemetry|istio-policy\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Heap in-use", - "refId": "E", - "step": 2 - }, - { - "expr": "go_memstats_stack_inuse_bytes{job=~\"istio-policy|istio-telemetry\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Stack in-use", - "refId": "G", - "step": 2 - }, - { - "expr": "sum(container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"})", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Total (k8s)", - "refId": "C", - "step": 2 - }, - { - "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ container_name }} (k8s)", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 48 - }, - "id": 95, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Total (k8s)", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}[1m])) by (container_name)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ container_name }} (k8s)", - "refId": "B", - "step": 2 - }, - { - "expr": "irate(process_cpu_seconds_total{job=~\"istio-policy|istio-telemetry\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "mixer (self-reported)", - "refId": "C", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "vCPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 48 - }, - "id": 96, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_open_fds{job=~\"istio-policy|istio-telemetry\"}", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Open FDs (pilot)", - "refId": "A" - }, - { - "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ container_name }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "none", - "label": "", - "logBase": 1024, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 48 - }, - "id": 97, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{job=\"istio-telemetry\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Number of Goroutines", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "10s", - "schemaVersion": 18, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Istio Performance Dashboard", - "uid": "vu8e0VWZk", - "version": 22 -} -' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-istio-service-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - istio-service-dashboard.json: '{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "iteration": 1536442501501, - "links": [], - "panels": [ - { - "content": "
\nSERVICE: $service\n
", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 89, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "Prometheus", - "format": "ops", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 3 - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\"}[5m])), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Client Request Volume", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "Prometheus", - "decimals": null, - "format": "percentunit", - "gauge": { - "maxValue": 100, - "minValue": 80, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 3 - }, - "id": 14, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"source\",destination_service=~\"$service\"}[5m]))", - "format": "time_series", - "intervalFactor": 1, - "refId": "B" - } - ], - "thresholds": "95, 99, 99.5", - "title": "Client Success Rate (non-5xx responses)", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 3 - }, - "id": 87, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "P50", - "refId": "A" - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "P90", - "refId": "B" - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\",destination_service=~\"$service\"}[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "P99", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Client Request Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 3 - }, - "id": 84, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", destination_service=~\"$service\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "TCP Received Bytes", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "Prometheus", - "format": "ops", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 7 - }, - "id": 97, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[5m])), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Server Request Volume", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "Prometheus", - "decimals": null, - "format": "percentunit", - "gauge": { - "maxValue": 100, - "minValue": 80, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 6, - "y": 7 - }, - "id": 98, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"destination\",destination_service=~\"$service\"}[5m]))", - "format": "time_series", - "intervalFactor": 1, - "refId": "B" - } - ], - "thresholds": "95, 99, 99.5", - "title": "Server Success Rate (non-5xx responses)", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 4, - "w": 6, - "x": 12, - "y": 7 - }, - "id": 99, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "P50", - "refId": "A" - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "P90", - "refId": "B" - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_service=~\"$service\"}[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "P99", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Server Request Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 18, - "y": 7 - }, - "id": 100, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"source\", destination_service=~\"$service\"}[1m])) ", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "TCP Sent Bytes", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "content": "
\nCLIENT WORKLOADS\n
", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 45, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 25, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=\"source\",source_workload=~\"$srcwl\",source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }} (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=\"source\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Requests by Source And Response Code", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 26, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Success Rate (non-5xx responses) By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1.01", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 27, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Request Duration by Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 28, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Request Size By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 68, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Size By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 80, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Received from Incoming TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Sent to Incoming TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "
\nSERVICE WORKLOADS\n
", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 69, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 90, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\",destination_service=~\"$service\",reporter=\"destination\",destination_workload=~\"$dstwl\",destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace, response_code), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} : {{ response_code }} (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", reporter=\"destination\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace, response_code), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} : {{ response_code }}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Requests by Destination And Response Code", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 91, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\",response_code!~\"5.*\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[5m])) by (destination_workload, destination_workload_namespace)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Success Rate (non-5xx responses) By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1.01", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 41 - }, - "id": 94, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Request Duration by Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 41 - }, - "id": 95, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Request Size By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 41 - }, - "id": 96, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace }} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Size By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 92, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace}} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{ destination_workload_namespace}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Received from Incoming TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 93, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{destination_workload_namespace }} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", destination_service=~\"$service\", destination_workload=~\"$dstwl\", destination_workload_namespace=~\"$dstns\"}[1m])) by (destination_workload, destination_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_workload }}.{{destination_workload_namespace }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Sent to Incoming TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "datasource": "Prometheus", - "hide": 0, - "includeAll": false, - "label": "Service", - "multi": false, - "name": "service", - "options": [], - "query": "label_values(destination_service)", - "refresh": 1, - "regex": "", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Client Workload Namespace", - "multi": true, - "name": "srcns", - "options": [], - "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=\"$service\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\"}) by (source_workload_namespace))", - "refresh": 1, - "regex": "/.*namespace=\"([^\"]*).*/", - "sort": 2, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Client Workload", - "multi": true, - "name": "srcwl", - "options": [], - "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=~\"$service\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", - "refresh": 1, - "regex": "/.*workload=\"([^\"]*).*/", - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Service Workload Namespace", - "multi": true, - "name": "dstns", - "options": [], - "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=\"$service\"}) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\"}) by (destination_workload_namespace))", - "refresh": 1, - "regex": "/.*namespace=\"([^\"]*).*/", - "sort": 2, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Service Workload", - "multi": true, - "name": "dstwl", - "options": [], - "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_service=~\"$service\", destination_workload_namespace=~\"$dstns\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_service=~\"$service\", destination_workload_namespace=~\"$dstns\"}) by (destination_workload))", - "refresh": 1, - "regex": "/.*workload=\"([^\"]*).*/", - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Istio Service Dashboard", - "uid": "LJ_uJAvmk", - "version": 1 -} -' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-istio-workload-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - istio-workload-dashboard.json: '{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.0.4" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "text", - "name": "Text", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 0, - "id": null, - "iteration": 1531345461465, - "links": [], - "panels": [ - { - "content": "
\nWORKLOAD: $workload.$namespace\n
", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 89, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "Prometheus", - "format": "ops", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 0, - "y": 3 - }, - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\"}[5m])), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "refId": "A", - "step": 4 - } - ], - "thresholds": "", - "title": "Incoming Request Volume", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "datasource": "Prometheus", - "decimals": null, - "format": "percentunit", - "gauge": { - "maxValue": 100, - "minValue": 80, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": false - }, - "gridPos": { - "h": 4, - "w": 8, - "x": 8, - "y": 3 - }, - "id": 14, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\",response_code!~\"5.*\"}[5m])) / sum(irate(istio_requests_total{reporter=\"destination\",destination_workload_namespace=~\"$namespace\",destination_workload=~\"$workload\"}[5m]))", - "format": "time_series", - "intervalFactor": 1, - "refId": "B" - } - ], - "thresholds": "95, 99, 99.5", - "title": "Incoming Success Rate (non-5xx responses)", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 4, - "w": 8, - "x": 16, - "y": 3 - }, - "id": 87, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "P50", - "refId": "A" - }, - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "P90", - "refId": "B" - }, - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\",destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\"}[1m])) by (le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "P99", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Request Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 84, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\"}[1m])) + sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "TCP Server Traffic", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "datasource": "Prometheus", - "format": "Bps", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 85, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": true, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(irate(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\"}[1m])) + sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "", - "title": "TCP Client Traffic", - "transparent": false, - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "avg" - }, - { - "content": "
\nINBOUND WORKLOADS\n
", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 45, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 25, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", reporter=\"destination\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }} (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", reporter=\"destination\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace, response_code), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} : {{ response_code }}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Requests by Source And Response Code", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 26, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\",response_code!~\"5.*\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace) / sum(irate(istio_requests_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[5m])) by (source_workload, source_workload_namespace)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Success Rate (non-5xx responses) By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1.01", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 20 - }, - "id": 27, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Request Duration by Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 28, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Request Size By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 68, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload=~\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{source_workload}}.{{source_workload_namespace}} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Size By Source", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 80, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", connection_security_policy=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"destination\", connection_security_policy!=\"mutual_tls\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Received from Incoming TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 82, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"destination\", destination_workload_namespace=~\"$namespace\", destination_workload=~\"$workload\", source_workload=~\"$srcwl\", source_workload_namespace=~\"$srcns\"}[1m])) by (source_workload, source_workload_namespace), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ source_workload }}.{{ source_workload_namespace}}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Sent to Incoming TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "content": "
\nOUTBOUND SERVICES\n
", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 69, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 70, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} : {{ response_code }} (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_requests_total{connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", reporter=\"source\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service, response_code), 0.001)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} : {{ response_code }}", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Outgoing Requests by Destination And Response Code", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 71, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\",response_code!~\"5.*\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service) / sum(irate(istio_requests_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[5m])) by (destination_service)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{destination_service }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Outgoing Success Rate (non-5xx responses) By Destination", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": "1.01", - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 41 - }, - "id": 72, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.50, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.50, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.90, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.90, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.95, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.95, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "(histogram_quantile(0.99, sum(irate(istio_request_duration_milliseconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le)) / 1000) or histogram_quantile(0.99, sum(irate(istio_request_duration_seconds_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Outgoing Request Duration by Destination", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 41 - }, - "id": 73, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_request_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Outgoing Request Size By Destination", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 41 - }, - "id": 74, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "hideEmpty": true, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P50 (🔐mTLS)", - "refId": "D", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P90 (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P95 (🔐mTLS)", - "refId": "B", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P99 (🔐mTLS)", - "refId": "C", - "step": 2 - }, - { - "expr": "histogram_quantile(0.50, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P50", - "refId": "E", - "step": 2 - }, - { - "expr": "histogram_quantile(0.90, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P90", - "refId": "F", - "step": 2 - }, - { - "expr": "histogram_quantile(0.95, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P95", - "refId": "G", - "step": 2 - }, - { - "expr": "histogram_quantile(0.99, sum(irate(istio_response_bytes_bucket{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service, le))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} P99", - "refId": "H", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Size By Destination", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 76, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy=\"mutual_tls\", reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_sent_bytes_total{connection_security_policy!=\"mutual_tls\", reporter=\"source\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_service }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Sent on Outgoing TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 78, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_service }} (🔐mTLS)", - "refId": "A", - "step": 2 - }, - { - "expr": "round(sum(irate(istio_tcp_received_bytes_total{reporter=\"source\", connection_security_policy!=\"mutual_tls\", source_workload_namespace=~\"$namespace\", source_workload=~\"$workload\", destination_service=~\"$dstsvc\"}[1m])) by (destination_service), 0.001)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ destination_service }}", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Bytes Received from Outgoing TCP Connection", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ] - } - ], - "refresh": "10s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "hide": 0, - "includeAll": false, - "label": "Namespace", - "multi": false, - "name": "namespace", - "options": [], - "query": "query_result(sum(istio_requests_total) by (destination_workload_namespace) or sum(istio_tcp_sent_bytes_total) by (destination_workload_namespace))", - "refresh": 1, - "regex": "/.*_namespace=\"([^\"]*).*/", - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "hide": 0, - "includeAll": false, - "label": "Workload", - "multi": false, - "name": "workload", - "options": [], - "query": "query_result((sum(istio_requests_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_requests_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)) or (sum(istio_tcp_sent_bytes_total{destination_workload_namespace=~\"$namespace\"}) by (destination_workload) or sum(istio_tcp_sent_bytes_total{source_workload_namespace=~\"$namespace\"}) by (source_workload)))", - "refresh": 1, - "regex": "/.*workload=\"([^\"]*).*/", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Inbound Workload Namespace", - "multi": true, - "name": "srcns", - "options": [], - "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\"}) by (source_workload_namespace))", - "refresh": 1, - "regex": "/.*namespace=\"([^\"]*).*/", - "sort": 2, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Inbound Workload", - "multi": true, - "name": "srcwl", - "options": [], - "query": "query_result( sum(istio_requests_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload) or sum(istio_tcp_sent_bytes_total{reporter=\"destination\", destination_workload=\"$workload\", destination_workload_namespace=~\"$namespace\", source_workload_namespace=~\"$srcns\"}) by (source_workload))", - "refresh": 1, - "regex": "/.*workload=\"([^\"]*).*/", - "sort": 3, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Destination Service", - "multi": true, - "name": "dstsvc", - "options": [], - "query": "query_result( sum(istio_requests_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service) or sum(istio_tcp_sent_bytes_total{reporter=\"source\", source_workload=~\"$workload\", source_workload_namespace=~\"$namespace\"}) by (destination_service))", - "refresh": 1, - "regex": "/.*destination_service=\"([^\"]*).*/", - "sort": 4, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Istio Workload Dashboard", - "uid": "UbsSZTDik", - "version": 1 -} -' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-mixer-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - mixer-dashboard.json: '{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.2.3" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "text", - "name": "Text", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "limit": 100, - "name": "Annotations & Alerts", - "showIn": 0, - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1543881232533, - "links": [], - "panels": [ - { - "content": "

Deployed Versions

", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 0 - }, - "height": "40", - "id": 62, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 3 - }, - "id": 64, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(istio_build{component=\"mixer\"}) by (tag)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ tag }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Mixer Versions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "

Resource Usage

", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 8 - }, - "height": "40", - "id": 29, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 11 - }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(process_virtual_memory_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "Virtual Memory ({{ job }})", - "refId": "I" - }, - { - "expr": "sum(process_resident_memory_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Resident Memory ({{ job }})", - "refId": "H" - }, - { - "expr": "sum(go_memstats_heap_sys_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap sys ({{ job }})", - "refId": "A" - }, - { - "expr": "sum(go_memstats_heap_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap alloc ({{ job }})", - "refId": "D" - }, - { - "expr": "sum(go_memstats_alloc_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Alloc ({{ job }})", - "refId": "F" - }, - { - "expr": "sum(go_memstats_heap_inuse_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Heap in-use ({{ job }})", - "refId": "E" - }, - { - "expr": "sum(go_memstats_stack_inuse_bytes{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Stack in-use ({{ job }})", - "refId": "G" - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}, \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")) by (service)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ service }} total (k8s)", - "refId": "C" - }, - { - "expr": "sum(label_replace(container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}, \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")) by (container_name, service)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ service }} - {{ container_name }} (k8s)", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 11 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace(sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}[1m])) by (pod_name), \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ service }} total (k8s)", - "refId": "A" - }, - { - "expr": "label_replace(sum(rate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}[1m])) by (container_name, pod_name), \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ service }} - {{ container_name }} (k8s)", - "refId": "B" - }, - { - "expr": "sum(irate(process_cpu_seconds_total{job=~\"istio-telemetry|istio-policy\"}[1m])) by (job)", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "{{ job }} (self-reported)", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 11 - }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(process_open_fds{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "hide": true, - "instant": false, - "interval": "", - "intervalFactor": 2, - "legendFormat": "Open FDs ({{ job }})", - "refId": "A" - }, - { - "expr": "sum(label_replace(container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"mixer|istio-proxy\", pod_name=~\"istio-telemetry-.*|istio-policy-.*\"}, \"service\", \"$1\" , \"pod_name\", \"(istio-telemetry|istio-policy)-.*\")) by (container_name, service)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ service }} - {{ container_name }}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Disk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "none", - "label": "", - "logBase": 1024, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 11 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(go_goroutines{job=~\"istio-telemetry|istio-policy\"}) by (job)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Number of Goroutines ({{ job }})", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "

Mixer Overview

", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 18 - }, - "height": "40px", - "id": 30, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 21 - }, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_io_server_completed_rpcs[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "mixer (Total)", - "refId": "B" - }, - { - "expr": "sum(rate(grpc_io_server_completed_rpcs[1m])) by (grpc_server_method)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "mixer ({{ grpc_server_method }})", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Incoming Requests", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 21 - }, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "{}", - "yaxis": 1 - } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.5, sum(rate(grpc_io_server_server_latency_bucket{}[1m])) by (grpc_server_method, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ grpc_server_method }} 0.5", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.9, sum(rate(grpc_io_server_server_latency_bucket{}[1m])) by (grpc_server_method, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ grpc_server_method }} 0.9", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(grpc_io_server_server_latency_bucket{}[1m])) by (grpc_server_method, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ grpc_server_method }} 0.99", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Response Durations", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 21 - }, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(grpc_server_handled_total{grpc_code=~\"Unknown|Unimplemented|Internal|DataLoss\"}[1m])) by (grpc_method)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Mixer {{ grpc_method }}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Server Error Rate (5xx responses)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 21 - }, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(grpc_server_handled_total{grpc_code!=\"OK\",grpc_service=~\".*Mixer\"}[1m])) by (grpc_method)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Mixer {{ grpc_method }}", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Non-successes (4xxs)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "

Adapters and Config

", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 28, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 30 - }, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(mixer_runtime_dispatches_total{adapter=~\"$adapter\"}[1m])) by (adapter)", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ adapter }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Adapter Dispatch Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 30 - }, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.5, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (adapter, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ adapter }} - p50", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.9, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (adapter, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ adapter }} - p90 ", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (adapter, le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ adapter }} - p99", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Adapter Dispatch Duration", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 37 - }, - "id": 60, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(topk(1, max(mixer_config_rule_config_count) by (configID)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Rules", - "refId": "A" - }, - { - "expr": "scalar(topk(1, max(mixer_config_rule_config_error_count) by (configID)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Config Errors", - "refId": "B" - }, - { - "expr": "scalar(topk(1, max(mixer_config_rule_config_match_error_count) by (configID)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Match Errors", - "refId": "C" - }, - { - "expr": "scalar(topk(1, max(mixer_config_unsatisfied_action_handler_count) by (configID)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Unsatisfied Actions", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Rules", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 37 - }, - "id": 56, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(topk(1, max(mixer_config_instance_config_count) by (configID)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Instances", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Instances in Latest Config", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 37 - }, - "id": 54, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(topk(1, max(mixer_config_handler_config_count) by (configID)))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Handlers", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Handlers in Latest Config", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 37 - }, - "id": 58, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "scalar(topk(1, max(mixer_config_attribute_count) by (configID)))", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "Attributes", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Attributes in Latest Config", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "content": "

Individual Adapters

", - "gridPos": { - "h": 3, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 23, - "links": [], - "mode": "html", - "title": "", - "transparent": true, - "type": "text" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 47 - }, - "id": 46, - "panels": [], - "repeat": "adapter", - "title": "$adapter Adapter", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace(irate(mixer_runtime_dispatches_total{adapter=~\"$adapter\"}[1m]),\"handler\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ handler }} (error: {{ error }})", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Dispatch Count By Handler", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "label_replace(histogram_quantile(0.5, sum(rate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (handler, error, le)), \"handler_short\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "p50 - {{ handler_short }} (error: {{ error }})", - "refId": "A" - }, - { - "expr": "label_replace(histogram_quantile(0.9, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (handler, error, le)), \"handler_short\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "p90 - {{ handler_short }} (error: {{ error }})", - "refId": "D" - }, - { - "expr": "label_replace(histogram_quantile(0.99, sum(irate(mixer_runtime_dispatch_duration_seconds_bucket{adapter=~\"$adapter\"}[1m])) by (handler, error, le)), \"handler_short\", \"$1 ($3)\", \"handler\", \"(.*)\\\\.(.*)\\\\.(.*)\")", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "p99 - {{ handler_short }} (error: {{ error }})", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "Dispatch Duration By Handler", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 16, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": {}, - "datasource": "Prometheus", - "hide": 0, - "includeAll": true, - "label": "Adapter", - "multi": true, - "name": "adapter", - "options": [], - "query": "label_values(adapter)", - "refresh": 2, - "regex": "", - "sort": 1, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Istio Mixer Dashboard", - "version": 4 -} -' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana-configuration-dashboards-pilot-dashboard - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - pilot-dashboard.json: '{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": false, - "gnetId": null, - "graphTooltip": 1, - "id": 11, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 60, - "panels": [], - "title": "Deployed Versions", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 1 - }, - "id": 56, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(istio_build{component=\"pilot\"}) by (tag)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{ tag }}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Pilot Versions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 62, - "panels": [], - "title": "Resource Usage", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 7 - }, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "process_virtual_memory_bytes{job=\"pilot\"}", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "Virtual Memory", - "refId": "I", - "step": 2 - }, - { - "expr": "process_resident_memory_bytes{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Resident Memory", - "refId": "H", - "step": 2 - }, - { - "expr": "go_memstats_heap_sys_bytes{job=\"pilot\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap sys", - "refId": "A" - }, - { - "expr": "go_memstats_heap_alloc_bytes{job=\"pilot\"}", - "format": "time_series", - "hide": true, - "intervalFactor": 2, - "legendFormat": "heap alloc", - "refId": "D" - }, - { - "expr": "go_memstats_alloc_bytes{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Alloc", - "refId": "F", - "step": 2 - }, - { - "expr": "go_memstats_heap_inuse_bytes{job=\"pilot\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Heap in-use", - "refId": "E", - "step": 2 - }, - { - "expr": "go_memstats_stack_inuse_bytes{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Stack in-use", - "refId": "G", - "step": 2 - }, - { - "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"discovery\", pod_name=~\"istio-pilot-.*\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Discovery (container)", - "refId": "B", - "step": 2 - }, - { - "expr": "container_memory_usage_bytes{job=\"kubernetes-cadvisor\", container_name=~\"istio-proxy\", pod_name=~\"istio-pilot-.*\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Sidecar (container)", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Memory", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 7 - }, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=\"discovery\", pod_name=~\"istio-pilot-.*\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Discovery (container)", - "refId": "A" - }, - { - "expr": "irate(process_cpu_seconds_total{job=\"pilot\"}[1m])", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Discovery (process)", - "refId": "C", - "step": 2 - }, - { - "expr": "sum(irate(container_cpu_usage_seconds_total{job=\"kubernetes-cadvisor\",container_name=\"istio-proxy\", pod_name=~\"istio-pilot-.*\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 2, - "legendFormat": "Sidecar (container)", - "refId": "B", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "CPU", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 7 - }, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=\"discovery\", pod_name=~\"istio-pilot-.*\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Discovery", - "refId": "B", - "step": 2 - }, - { - "expr": "container_fs_usage_bytes{job=\"kubernetes-cadvisor\", container_name=\"istio-proxy\", pod_name=~\"istio-pilot-.*\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Sidecar", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Disk", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "none", - "label": "", - "logBase": 1024, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 7 - }, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Number of Goroutines", - "refId": "A", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 14 - }, - "id": 58, - "panels": [], - "title": "Pilot Push Information", - "type": "row" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "description": "Shows the rate of pilot pushes", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 15 - }, - "id": 622, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "paceLength": 10, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(pilot_xds_pushes{type=\"cds\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Cluster", - "refId": "C" - }, - { - "expr": "sum(irate(pilot_xds_pushes{type=\"eds\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Endpoints", - "refId": "D" - }, - { - "expr": "sum(irate(pilot_xds_pushes{type=\"lds\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Listeners", - "refId": "A" - }, - { - "expr": "sum(irate(pilot_xds_pushes{type=\"rds\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Routes", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Pilot Pushes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Captures a variety of pilot errors", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 15 - }, - "id": 67, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(pilot_xds_cds_reject{job=\"pilot\"}) or (absent(pilot_xds_cds_reject{job=\"pilot\"}) - 1)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Rejected CDS Configs", - "refId": "C" - }, - { - "expr": "sum(pilot_xds_eds_reject{job=\"pilot\"}) or (absent(pilot_xds_eds_reject{job=\"pilot\"}) - 1)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Rejected EDS Configs", - "refId": "D" - }, - { - "expr": "sum(pilot_xds_rds_reject{job=\"pilot\"}) or (absent(pilot_xds_rds_reject{job=\"pilot\"}) - 1)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Rejected RDS Configs", - "refId": "A" - }, - { - "expr": "sum(pilot_xds_lds_reject{job=\"pilot\"}) or (absent(pilot_xds_lds_reject{job=\"pilot\"}) - 1)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Rejected LDS Configs", - "refId": "B" - }, - { - "expr": "sum(rate(pilot_xds_write_timeout{job=\"pilot\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Write Timeouts", - "refId": "F" - }, - { - "expr": "sum(rate(pilot_total_xds_internal_errors{job=\"pilot\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Internal Errors", - "refId": "H" - }, - { - "expr": "sum(rate(pilot_total_xds_rejects{job=\"pilot\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Config Rejection Rate", - "refId": "E" - }, - { - "expr": "sum(rate(pilot_xds_push_context_errors{job=\"pilot\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Push Context Errors", - "refId": "K" - }, - { - "expr": "sum(rate(pilot_xds_pushes{type!~\"lds|cds|rds|eds\"}[1m])) by (type)", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Push Errors ({{ type }})", - "refId": "L" - }, - { - "expr": "sum(rate(pilot_xds_push_errors{job=\"pilot\"}[1m])) by (type)", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Push Errors ({{ type }})", - "refId": "I" - }, - { - "expr": "sum(rate(pilot_xds_push_timeout{job=\"pilot\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Push Timeouts", - "refId": "G" - }, - { - "expr": "sum(rate(pilot_xds_push_timeout_failures{job=\"pilot\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Push Timeouts Failures", - "refId": "J" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Pilot Errors", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "description": "Shows the total time it takes to push a config update to a proxy", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 15 - }, - "id": 624, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.5, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p50 ", - "refId": "A" - }, - { - "expr": "histogram_quantile(0.9, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p90", - "refId": "B" - }, - { - "expr": "histogram_quantile(0.99, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p99", - "refId": "C" - }, - { - "expr": "histogram_quantile(0.999, sum(rate(pilot_proxy_convergence_time_bucket[1m])) by (le))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "p99.9", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Proxy Push Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 23 - }, - "id": 45, - "legend": { - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pilot_conflict_inbound_listener{job=\"pilot\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Inbound Listeners", - "refId": "B" - }, - { - "expr": "pilot_conflict_outbound_listener_http_over_current_tcp{job=\"pilot\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Outbound Listeners (http over current tcp)", - "refId": "A" - }, - { - "expr": "pilot_conflict_outbound_listener_tcp_over_current_tcp{job=\"pilot\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Outbound Listeners (tcp over current tcp)", - "refId": "C" - }, - { - "expr": "pilot_conflict_outbound_listener_tcp_over_current_http{job=\"pilot\"}", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "Outbound Listeners (tcp over current http)", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Conflicts", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 23 - }, - "id": 47, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "pilot_virt_services{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Virtual Services", - "refId": "A" - }, - { - "expr": "pilot_services{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Services", - "refId": "B" - }, - { - "expr": "pilot_xds{job=\"pilot\"}", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Connected Endpoints", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "ADS Monitoring", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "columns": [], - "datasource": "Prometheus", - "description": "Clusters in this table do not have any endpoints known to pilot. This could be from referencing subsets that do not have any instances, or pods marked as NotReady", - "fontSize": "100%", - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 23 - }, - "id": 51, - "links": [], - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": null, - "desc": false - }, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "pattern": "Time", - "type": "date" - }, - { - "alias": "Clusters", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "decimals": 2, - "pattern": "/.*/", - "thresholds": [], - "type": "number", - "unit": "short" - } - ], - "targets": [ - { - "expr": "sum(pilot_xds_eds_instances{job=\"pilot\", cluster=~\".+\\\\|.+\"}) by (cluster) < 1", - "format": "time_series", - "hide": false, - "instant": true, - "intervalFactor": 1, - "legendFormat": "{{cluster}}", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Clusters with no known endpoints", - "transform": "timeseries_aggregations", - "type": "table" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 64, - "panels": [], - "title": "Envoy Information", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Shows details about Envoy proxies in the mesh", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 32 - }, - "id": 40, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(irate(envoy_cluster_upstream_cx_total{cluster_name=\"xds-grpc\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "XDS Connections", - "refId": "C" - }, - { - "expr": "sum(irate(envoy_cluster_upstream_cx_connect_fail{cluster_name=\"xds-grpc\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "XDS Connection Failures", - "refId": "A" - }, - { - "expr": "sum(increase(envoy_server_hot_restart_epoch[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "Envoy Restarts", - "refId": "B" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Envoy Details", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 32 - }, - "id": 41, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(envoy_cluster_upstream_cx_active{cluster_name=\"xds-grpc\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "XDS Active Connections", - "refId": "C", - "step": 2 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "XDS Active Connections", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "description": "Shows the size of XDS requests and responses", - "fill": 1, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 32 - }, - "id": 42, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max(rate(envoy_cluster_upstream_cx_rx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "XDS Response Bytes Max", - "refId": "D" - }, - { - "expr": "quantile(0.5, rate(envoy_cluster_upstream_cx_rx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", - "format": "time_series", - "hide": false, - "intervalFactor": 1, - "legendFormat": "XDS Response Bytes Average", - "refId": "B" - }, - { - "expr": "max(rate(envoy_cluster_upstream_cx_tx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "XDS Request Bytes Max", - "refId": "A" - }, - { - "expr": "quantile(.5, rate(envoy_cluster_upstream_cx_tx_bytes_total{cluster_name=\"xds-grpc\"}[1m]))", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "XDS Request Bytes Average", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "XDS Requests Size", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "5s", - "schemaVersion": 18, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "browser", - "title": "Istio Pilot Dashboard", - "uid": "3--MLVZZk", - "version": 11 -}' ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-grafana - namespace: istio-system - labels: - app: grafana - release: istio - istio: grafana -data: - datasources.yaml: | - apiVersion: 1 - datasources: - - access: proxy - editable: true - isDefault: true - jsonData: - timeInterval: 5s - name: Prometheus - orgId: 1 - type: prometheus - url: http://prometheus:9090 - - dashboardproviders.yaml: | - apiVersion: 1 - providers: - - disableDeletion: false - folder: istio - name: istio - options: - path: /var/lib/grafana/dashboards/istio - orgId: 1 - type: file ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: grafana - namespace: istio-system - labels: - app: grafana - release: istio -spec: - replicas: 1 - selector: - matchLabels: - app: grafana - template: - metadata: - labels: - app: grafana - chart: grafana - heritage: Tiller - release: istio-system - annotations: - sidecar.istio.io/inject: "false" - spec: - securityContext: - runAsUser: 472 - fsGroup: 472 - containers: - - name: grafana - image: "grafana/grafana:6.4.3" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 3000 - readinessProbe: - httpGet: - path: /api/health - port: 3000 - env: - - name: GRAFANA_PORT - value: "3000" - - name: GF_AUTH_BASIC_ENABLED - value: "false" - - name: GF_AUTH_ANONYMOUS_ENABLED - value: "true" - - name: GF_AUTH_ANONYMOUS_ORG_ROLE - value: Admin - - name: GF_PATHS_DATA - value: /data/grafana - resources: - requests: - cpu: 10m - - volumeMounts: - - name: data - mountPath: /data/grafana - - name: dashboards-istio-citadel-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/citadel-dashboard.json" - subPath: citadel-dashboard.json - readOnly: true - - name: dashboards-istio-galley-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/galley-dashboard.json" - subPath: galley-dashboard.json - readOnly: true - - name: dashboards-istio-istio-mesh-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/istio-mesh-dashboard.json" - subPath: istio-mesh-dashboard.json - readOnly: true - - name: dashboards-istio-istio-performance-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/istio-performance-dashboard.json" - subPath: istio-performance-dashboard.json - readOnly: true - - name: dashboards-istio-istio-service-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/istio-service-dashboard.json" - subPath: istio-service-dashboard.json - readOnly: true - - name: dashboards-istio-istio-workload-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/istio-workload-dashboard.json" - subPath: istio-workload-dashboard.json - readOnly: true - - name: dashboards-istio-mixer-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/mixer-dashboard.json" - subPath: mixer-dashboard.json - readOnly: true - - name: dashboards-istio-pilot-dashboard - mountPath: "/var/lib/grafana/dashboards/istio/pilot-dashboard.json" - subPath: pilot-dashboard.json - readOnly: true - - name: config - mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml" - subPath: datasources.yaml - - name: config - mountPath: "/etc/grafana/provisioning/dashboards/dashboardproviders.yaml" - subPath: dashboardproviders.yaml - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - volumes: - - name: config - configMap: - name: istio-grafana - - name: data - emptyDir: {} - - name: dashboards-istio-citadel-dashboard - configMap: - name: istio-grafana-configuration-dashboards-citadel-dashboard - - name: dashboards-istio-galley-dashboard - configMap: - name: istio-grafana-configuration-dashboards-galley-dashboard - - name: dashboards-istio-istio-mesh-dashboard - configMap: - name: istio-grafana-configuration-dashboards-istio-mesh-dashboard - - name: dashboards-istio-istio-performance-dashboard - configMap: - name: istio-grafana-configuration-dashboards-istio-performance-dashboard - - name: dashboards-istio-istio-service-dashboard - configMap: - name: istio-grafana-configuration-dashboards-istio-service-dashboard - - name: dashboards-istio-istio-workload-dashboard - configMap: - name: istio-grafana-configuration-dashboards-istio-workload-dashboard - - name: dashboards-istio-mixer-dashboard - configMap: - name: istio-grafana-configuration-dashboards-mixer-dashboard - - name: dashboards-istio-pilot-dashboard - configMap: - name: istio-grafana-configuration-dashboards-pilot-dashboard ---- - - -apiVersion: authentication.istio.io/v1alpha1 -kind: Policy -metadata: - name: grafana-ports-mtls-disabled - namespace: istio-system - labels: - app: grafana - release: istio -spec: - targets: - - name: grafana - ports: - - number: 3000 ---- - - -apiVersion: v1 -kind: Service -metadata: - name: grafana - namespace: istio-system - annotations: - labels: - app: grafana - release: istio -spec: - type: ClusterIP - ports: - - port: 3000 - targetPort: 3000 - protocol: TCP - name: http - selector: - app: grafana ---- - -# Resources for IngressGateway component - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: istio-ingressgateway - istio: ingressgateway - release: istio - name: istio-ingressgateway - namespace: istio-system -spec: - selector: - matchLabels: - app: istio-ingressgateway - istio: ingressgateway - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: istio-ingressgateway - chart: gateways - heritage: Tiller - istio: ingressgateway - release: istio - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - args: - - proxy - - router - - --domain - - $(POD_NAMESPACE).svc.cluster.local - - --proxyLogLevel=warning - - --proxyComponentLogLevel=misc:error - - --log_output_level=default:info - - --drainDuration - - 45s - - --parentShutdownDuration - - 1m0s - - --connectTimeout - - 10s - - --serviceCluster - - istio-ingressgateway - - --zipkinAddress - - zipkin.istio-system:9411 - - --proxyAdminPort - - "15000" - - --statusPort - - "15020" - - --controlPlaneAuthPolicy - - NONE - - --discoveryAddress - - istio-pilot.istio-system:15010 - - --trust-domain=cluster.local - env: - - name: NODE_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: INSTANCE_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.podIP - - name: HOST_IP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - name: SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: ISTIO_META_WORKLOAD_NAME - value: istio-ingressgateway - - name: ISTIO_META_OWNER - value: kubernetes://api/apps/v1/namespaces/istio-system/deployments/istio-ingressgateway - - name: ISTIO_META_MESH_ID - value: cluster.local - - name: ISTIO_META_POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: ISTIO_META_CONFIG_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: ISTIO_META_ROUTER_MODE - value: sni-dnat - - name: ISTIO_METAJSON_LABELS - value: | - {"app":"istio-ingressgateway","istio":"ingressgateway"} - - name: ISTIO_META_CLUSTER_ID - value: Kubernetes - - name: SDS_ENABLED - value: "false" - image: docker.io/istio/proxyv2:1.4.2 - imagePullPolicy: IfNotPresent - name: istio-proxy - ports: - - containerPort: 15020 - - containerPort: 80 - - containerPort: 443 - - containerPort: 15029 - - containerPort: 15030 - - containerPort: 15031 - - containerPort: 15032 - - containerPort: 15443 - - containerPort: 15011 - - containerPort: 8060 - - containerPort: 853 - - containerPort: 15090 - name: http-envoy-prom - protocol: TCP - readinessProbe: - failureThreshold: 30 - httpGet: - path: /healthz/ready - port: 15020 - scheme: HTTP - initialDelaySeconds: 1 - periodSeconds: 2 - successThreshold: 1 - timeoutSeconds: 1 - resources: - limits: - cpu: 2000m - memory: 1024Mi - requests: - cpu: 10m - memory: 40Mi - volumeMounts: - - mountPath: /etc/certs - name: istio-certs - readOnly: true - - mountPath: /etc/istio/ingressgateway-certs - name: ingressgateway-certs - readOnly: true - - mountPath: /etc/istio/ingressgateway-ca-certs - name: ingressgateway-ca-certs - readOnly: true - serviceAccountName: istio-ingressgateway-service-account - volumes: - - name: istio-certs - secret: - optional: true - secretName: istio.istio-ingressgateway-service-account - - name: ingressgateway-certs - secret: - optional: true - secretName: istio-ingressgateway-certs - - name: ingressgateway-ca-certs - secret: - optional: true - secretName: istio-ingressgateway-ca-certs - ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: Gateway -metadata: - name: ingressgateway - namespace: istio-system - labels: - release: istio -spec: - selector: - istio: ingressgateway - servers: - - port: - number: 80 - name: http - protocol: HTTP - hosts: - - "*" - # Additional ports in gateaway for the ingressPorts - apps using dedicated port instead of hostname ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: ingressgateway - namespace: istio-system - labels: - app: istio-ingressgateway - release: istio - istio: ingressgateway -spec: - minAvailable: 1 - selector: - matchLabels: - app: istio-ingressgateway - release: istio - istio: ingressgateway ---- - - -apiVersion: v1 -kind: Service -metadata: - name: istio-ingressgateway - namespace: istio-system - annotations: - labels: - app: istio-ingressgateway - release: istio - istio: ingressgateway -spec: - type: LoadBalancer - selector: - app: istio-ingressgateway - ports: - - - name: status-port - port: 15020 - targetPort: 15020 - - - name: http2 - port: 80 - targetPort: 80 - nodePort: 31280 - - - name: https - port: 443 - - - name: kiali - port: 15029 - targetPort: 15029 - - - name: prometheus - port: 15030 - targetPort: 15030 - - - name: grafana - port: 15031 - targetPort: 15031 - - - name: tracing - port: 15032 - targetPort: 15032 - - - name: tls - port: 15443 - targetPort: 15443 ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-ingressgateway-service-account - namespace: istio-system - labels: - app: istio-ingressgateway - release: istio ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: Sidecar -metadata: - name: default - namespace: istio-system - labels: - release: istio -spec: - egress: - - hosts: - - "*/*" ---- - -# Resources for Injector component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-sidecar-injector-istio-system - labels: - app: sidecar-injector - release: istio - istio: sidecar-injector -rules: -- apiGroups: [""] - resources: ["configmaps"] - resourceNames: ["istio-sidecar-injector"] - verbs: ["get", "list", "watch"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - resourceNames: ["istio-sidecar-injector", "istio-sidecar-injector-istio-system"] - verbs: ["get", "list", "watch", "patch"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: istio-sidecar-injector-admin-role-binding-istio-system - labels: - app: sidecar-injector - release: istio - istio: sidecar-injector -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: istio-sidecar-injector-istio-system -subjects: - - kind: ServiceAccount - name: istio-sidecar-injector-service-account - namespace: istio-system ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: injector-mesh - namespace: istio-system - labels: - release: istio -data: - # This is the 'mesh' config, loaded by the sidecar injector. - # It is a different configmap from pilot to allow a-la-carte install of the injector and follow the model - # of reducing blast-radius of config changes and avoiding globals. - - # Note that injector uses a subset of the mesh config only - for clarity this is only generating the - # required config, i.e. the defaultConfig section. See injection-template .ProxyConfig settings. - - - mesh: |- - # Unix Domain Socket through which envoy communicates with NodeAgent SDS to get - # key/cert for mTLS. Use secret-mount files instead of SDS if set to empty. - sdsUdsPath: "" - - defaultConfig: - # - # TCP connection timeout between Envoy & the application, and between Envoys. - connectTimeout: 10s - # - ### ADVANCED SETTINGS ############# - # Where should envoy's configuration be stored in the istio-proxy container - configPath: "/etc/istio/proxy" - # The pseudo service name used for Envoy. - serviceCluster: istio-proxy - # These settings that determine how long an old Envoy - # process should be kept alive after an occasional reload. - drainDuration: 45s - parentShutdownDuration: 1m0s - # - # Port where Envoy listens (on local host) for admin commands - # You can exec into the istio-proxy container in a pod and - # curl the admin port (curl http://localhost:15000/) to obtain - # diagnostic information from Envoy. See - # https://lyft.github.io/envoy/docs/operations/admin.html - # for more details - proxyAdminPort: 15000 - # - # Set concurrency to a specific number to control the number of Proxy worker threads. - # If set to 0 (default), then start worker thread for each CPU thread/core. - concurrency: 2 - # - tracing: - zipkin: - # Address of the Zipkin collector - address: zipkin.istio-system:9411 - # - # Mutual TLS authentication between sidecars and istio control plane. - controlPlaneAuthPolicy: NONE - # - # Address where istio Pilot service is running - discoveryAddress: istio-pilot.istio-system:15010 ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: sidecarInjectorWebhook - istio: sidecar-injector - release: istio - name: istio-sidecar-injector - namespace: istio-system -spec: - replicas: 1 - selector: - matchLabels: - istio: sidecar-injector - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: sidecarInjectorWebhook - chart: sidecarInjectorWebhook - heritage: Tiller - istio: sidecar-injector - release: istio - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - args: - - --caCertFile=/etc/istio/certs/root-cert.pem - - --tlsCertFile=/etc/istio/certs/cert-chain.pem - - --tlsKeyFile=/etc/istio/certs/key.pem - - --injectConfig=/etc/istio/inject/config - - --meshConfig=/etc/istio/config/mesh - - --port=9443 - - --healthCheckInterval=2s - - --healthCheckFile=/tmp/health - - --reconcileWebhookConfig=true - - --webhookConfigName=istio-sidecar-injector - - --log_output_level=debug - image: docker.io/istio/sidecar_injector:1.4.2 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - /usr/local/bin/sidecar-injector - - probe - - --probe-path=/tmp/health - - --interval=4s - initialDelaySeconds: 4 - periodSeconds: 4 - name: sidecar-injector-webhook - readinessProbe: - exec: - command: - - /usr/local/bin/sidecar-injector - - probe - - --probe-path=/tmp/health - - --interval=4s - initialDelaySeconds: 4 - periodSeconds: 4 - resources: - requests: - cpu: 10m - volumeMounts: - - mountPath: /etc/istio/config - name: config-volume - readOnly: true - - mountPath: /etc/istio/certs - name: certs - readOnly: true - - mountPath: /etc/istio/inject - name: inject-config - readOnly: true - serviceAccountName: istio-sidecar-injector-service-account - volumes: - - configMap: - name: injector-mesh - name: config-volume - - name: certs - secret: - secretName: istio.istio-sidecar-injector-service-account - - configMap: - items: - - key: config - path: config - - key: values - path: values - name: istio-sidecar-injector - name: inject-config - ---- - - -apiVersion: admissionregistration.k8s.io/v1beta1 -kind: MutatingWebhookConfiguration -metadata: - name: istio-sidecar-injector - - labels: - app: sidecar-injector - release: istio -webhooks: - - name: sidecar-injector.istio.io - clientConfig: - service: - name: istio-sidecar-injector - namespace: istio-system - path: "/inject" - caBundle: "" - rules: - - operations: [ "CREATE" ] - apiGroups: [""] - apiVersions: ["v1"] - resources: ["pods"] - failurePolicy: Fail - namespaceSelector: - matchLabels: - istio-injection: enabled ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: istio-sidecar-injector - namespace: istio-system - labels: - app: sidecar-injector - release: istio - istio: sidecar-injector -spec: - minAvailable: 1 - selector: - matchLabels: - app: sidecar-injector - release: istio - istio: sidecar-injector ---- - - -apiVersion: v1 -kind: Service -metadata: - name: istio-sidecar-injector - namespace: istio-system - labels: - app: sidecarInjectorWebhook - release: istio - istio: sidecar-injector -spec: - ports: - - port: 443 - targetPort: 9443 - selector: - istio: sidecar-injector ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-sidecar-injector-service-account - namespace: istio-system - labels: - app: sidecarInjectorWebhook - release: istio - istio: sidecar-injector ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio-sidecar-injector - namespace: istio-system - labels: - release: istio - app: sidecar-injector - istio: sidecar-injector -data: - values: |- - {"certmanager":{"enabled":false,"hub":"quay.io/jetstack","image":"cert-manager-controller","namespace":"istio-system","tag":"v0.6.2"},"clusterResources":true,"cni":{"namespace":"istio-system"},"galley":{"enableAnalysis":false,"enabled":true,"image":"galley","namespace":"istio-system"},"gateways":{"istio-egressgateway":{"autoscaleEnabled":false,"enabled":true,"env":{"ISTIO_META_ROUTER_MODE":"sni-dnat"},"namespace":"istio-system","ports":[{"name":"http2","port":80},{"name":"https","port":443},{"name":"tls","port":15443,"targetPort":15443}],"secretVolumes":[{"mountPath":"/etc/istio/egressgateway-certs","name":"egressgateway-certs","secretName":"istio-egressgateway-certs"},{"mountPath":"/etc/istio/egressgateway-ca-certs","name":"egressgateway-ca-certs","secretName":"istio-egressgateway-ca-certs"}],"type":"ClusterIP","zvpn":{"enabled":true,"suffix":"global"}},"istio-ingressgateway":{"applicationPorts":"","autoscaleEnabled":false,"debug":"info","domain":"","enabled":true,"env":{"ISTIO_META_ROUTER_MODE":"sni-dnat"},"meshExpansionPorts":[{"name":"tcp-pilot-grpc-tls","port":15011,"targetPort":15011},{"name":"tcp-citadel-grpc-tls","port":8060,"targetPort":8060},{"name":"tcp-dns-tls","port":853,"targetPort":853}],"namespace":"istio-system","ports":[{"name":"status-port","port":15020,"targetPort":15020},{"name":"http2","port":80,"targetPort":80},{"name":"https","port":443},{"name":"kiali","port":15029,"targetPort":15029},{"name":"prometheus","port":15030,"targetPort":15030},{"name":"grafana","port":15031,"targetPort":15031},{"name":"tracing","port":15032,"targetPort":15032},{"name":"tls","port":15443,"targetPort":15443}],"sds":{"enabled":false,"image":"node-agent-k8s","resources":{"limits":{"cpu":"2000m","memory":"1024Mi"},"requests":{"cpu":"100m","memory":"128Mi"}}},"secretVolumes":[{"mountPath":"/etc/istio/ingressgateway-certs","name":"ingressgateway-certs","secretName":"istio-ingressgateway-certs"},{"mountPath":"/etc/istio/ingressgateway-ca-certs","name":"ingressgateway-ca-certs","secretName":"istio-ingressgateway-ca-certs"}],"type":"LoadBalancer","zvpn":{"enabled":true,"suffix":"global"}}},"global":{"arch":{"amd64":2,"ppc64le":2,"s390x":2},"certificates":[],"configNamespace":"istio-system","configValidation":true,"controlPlaneSecurityEnabled":false,"defaultNodeSelector":{},"defaultPodDisruptionBudget":{"enabled":true},"defaultResources":{"requests":{"cpu":"10m"}},"disablePolicyChecks":false,"enableHelmTest":false,"enableTracing":true,"enabled":true,"hub":"docker.io/istio","imagePullPolicy":"IfNotPresent","imagePullSecrets":[],"istioNamespace":"istio-system","k8sIngress":{"enableHttps":false,"enabled":false,"gatewayName":"ingressgateway"},"localityLbSetting":{"enabled":true},"logAsJson":false,"logging":{"level":"default:info"},"meshExpansion":{"enabled":false,"useILB":false},"meshNetworks":{},"mtls":{"auto":false,"enabled":false},"multiCluster":{"clusterName":"","enabled":false},"namespace":"istio-system","network":"","omitSidecarInjectorConfigMap":false,"oneNamespace":false,"operatorManageWebhooks":false,"outboundTrafficPolicy":{"mode":"ALLOW_ANY"},"policyCheckFailOpen":false,"policyNamespace":"istio-system","priorityClassName":"","prometheusNamespace":"istio-system","proxy":{"accessLogEncoding":"TEXT","accessLogFile":"/dev/stdout","accessLogFormat":"","autoInject":"enabled","clusterDomain":"cluster.local","componentLogLevel":"misc:error","concurrency":2,"dnsRefreshRate":"300s","enableCoreDump":false,"envoyAccessLogService":{"enabled":false},"envoyMetricsService":{"enabled":false,"tcpKeepalive":{"interval":"10s","probes":3,"time":"10s"},"tlsSettings":{"mode":"DISABLE","subjectAltNames":[]}},"envoyStatsd":{"enabled":false},"excludeIPRanges":"","excludeInboundPorts":"","excludeOutboundPorts":"","image":"proxyv2","includeIPRanges":"*","includeInboundPorts":"*","kubevirtInterfaces":"","logLevel":"warning","privileged":false,"protocolDetectionTimeout":"100ms","readinessFailureThreshold":30,"readinessInitialDelaySeconds":1,"readinessPeriodSeconds":2,"resources":{"limits":{"cpu":"2000m","memory":"1024Mi"},"requests":{"cpu":"10m","memory":"40Mi"}},"statusPort":15020,"tracer":"zipkin"},"proxy_init":{"image":"proxyv2","resources":{"limits":{"cpu":"100m","memory":"50Mi"},"requests":{"cpu":"10m","memory":"10Mi"}}},"sds":{"enabled":false,"token":{"aud":"istio-ca"},"udsPath":""},"securityNamespace":"istio-system","tag":"1.4.2","telemetryNamespace":"istio-system","tracer":{"datadog":{"address":"$(HOST_IP):8126"},"lightstep":{"accessToken":"","address":"","cacertPath":"","secure":true},"zipkin":{"address":""}},"trustDomain":"cluster.local","useMCP":true},"grafana":{"accessMode":"ReadWriteMany","contextPath":"/grafana","dashboardProviders":{"dashboardproviders.yaml":{"apiVersion":1,"providers":[{"disableDeletion":false,"folder":"istio","name":"istio","options":{"path":"/var/lib/grafana/dashboards/istio"},"orgId":1,"type":"file"}]}},"datasources":{"datasources.yaml":{"apiVersion":1}},"enabled":true,"env":{},"envSecrets":{},"image":{"repository":"grafana/grafana","tag":"6.4.3"},"ingress":{"enabled":false,"hosts":["grafana.local"]},"namespace":"istio-system","nodeSelector":{},"persist":false,"podAntiAffinityLabelSelector":[],"podAntiAffinityTermLabelSelector":[],"replicaCount":1,"security":{"enabled":false,"passphraseKey":"passphrase","secretName":"grafana","usernameKey":"username"},"service":{"annotations":{},"externalPort":3000,"name":"http","type":"ClusterIP"},"storageClassName":"","tolerations":[]},"istio_cni":{"enabled":false},"istiocoredns":{"coreDNSImage":"coredns/coredns","coreDNSPluginImage":"istio/coredns-plugin:0.2-istio-1.1","coreDNSTag":"1.6.2","enabled":false,"namespace":"istio-system"},"kiali":{"contextPath":"/kiali","createDemoSecret":true,"dashboard":{"passphraseKey":"passphrase","secretName":"kiali","usernameKey":"username","viewOnlyMode":false},"enabled":true,"hub":"quay.io/kiali","ingress":{"enabled":false,"hosts":["kiali.local"]},"namespace":"istio-system","nodeSelector":{},"podAntiAffinityLabelSelector":[],"podAntiAffinityTermLabelSelector":[],"replicaCount":1,"security":{"cert_file":"/kiali-cert/cert-chain.pem","enabled":false,"private_key_file":"/kiali-cert/key.pem"},"tag":"v1.9"},"mixer":{"adapters":{"kubernetesenv":{"enabled":true},"prometheus":{"enabled":true,"metricsExpiryDuration":"10m"},"stackdriver":{"auth":{"apiKey":"","appCredentials":false,"serviceAccountPath":""},"enabled":false,"tracer":{"enabled":false,"sampleProbability":1}},"stdio":{"enabled":true,"outputAsJson":false},"useAdapterCRDs":false},"policy":{"adapters":{"kubernetesenv":{"enabled":true},"useAdapterCRDs":false},"autoscaleEnabled":false,"enabled":true,"image":"mixer","namespace":"istio-system","sessionAffinityEnabled":false},"telemetry":{"autoscaleEnabled":false,"enabled":true,"env":{"GOMAXPROCS":"6"},"image":"mixer","loadshedding":{"latencyThreshold":"100ms","mode":"enforce"},"namespace":"istio-system","nodeSelector":{},"podAntiAffinityLabelSelector":[],"podAntiAffinityTermLabelSelector":[],"replicaCount":1,"reportBatchMaxEntries":100,"reportBatchMaxTime":"1s","sessionAffinityEnabled":false,"tolerations":[],"useMCP":true}},"nodeagent":{"enabled":false,"image":"node-agent-k8s","namespace":"istio-system"},"pilot":{"appNamespaces":[],"autoscaleEnabled":false,"autoscaleMax":5,"autoscaleMin":1,"configMap":true,"configNamespace":"istio-config","cpu":{"targetAverageUtilization":80},"enableProtocolSniffingForInbound":false,"enableProtocolSniffingForOutbound":true,"enabled":true,"env":{},"image":"pilot","ingress":{"ingressClass":"istio","ingressControllerMode":"OFF","ingressService":"istio-ingressgateway"},"keepaliveMaxServerConnectionAge":"30m","meshNetworks":{"networks":{}},"namespace":"istio-system","nodeSelector":{},"podAntiAffinityLabelSelector":[],"podAntiAffinityTermLabelSelector":[],"policy":{"enabled":false},"replicaCount":1,"tolerations":[],"traceSampling":1,"useMCP":true},"prometheus":{"contextPath":"/prometheus","enabled":true,"hub":"docker.io/prom","ingress":{"enabled":false,"hosts":["prometheus.local"]},"namespace":"istio-system","nodeSelector":{},"podAntiAffinityLabelSelector":[],"podAntiAffinityTermLabelSelector":[],"replicaCount":1,"retention":"6h","scrapeInterval":"15s","security":{"enabled":true},"tag":"v2.12.0","tolerations":[]},"security":{"dnsCerts":{"istio-pilot-service-account.istio-control":"istio-pilot.istio-control"},"enableNamespacesByDefault":true,"enabled":true,"image":"citadel","namespace":"istio-system","selfSigned":true,"trustDomain":"cluster.local"},"sidecarInjectorWebhook":{"alwaysInjectSelector":[],"enableNamespacesByDefault":false,"enabled":true,"image":"sidecar_injector","injectLabel":"istio-injection","injectedAnnotations":{},"namespace":"istio-system","neverInjectSelector":[],"nodeSelector":{},"objectSelector":{"autoInject":true,"enabled":false},"podAnnotations":{},"podAntiAffinityLabelSelector":[],"podAntiAffinityTermLabelSelector":[],"replicaCount":1,"resources":{},"rewriteAppHTTPProbe":false,"rollingMaxSurge":"100%","rollingMaxUnavailable":"25%","selfSigned":false,"tolerations":[]},"telemetry":{"enabled":true,"v2":{"enabled":false}},"tracing":{"enabled":true,"ingress":{"enabled":false},"jaeger":{"accessMode":"ReadWriteMany","enabled":true,"hub":"docker.io/jaegertracing","memory":{"max_traces":50000},"namespace":"istio-system","persist":false,"spanStorageType":"badger","storageClassName":"","tag":"1.14"},"nodeSelector":{},"opencensus":{"exporters":{"stackdriver":{"enable_tracing":true}},"hub":"docker.io/omnition","resources":{"limits":{"cpu":"1","memory":"2Gi"},"requests":{"cpu":"200m","memory":"400Mi"}},"tag":"0.1.9"},"podAntiAffinityLabelSelector":[],"podAntiAffinityTermLabelSelector":[],"provider":"jaeger","service":{"annotations":{},"externalPort":9411,"name":"http-query","type":"ClusterIP"},"zipkin":{"hub":"docker.io/openzipkin","javaOptsHeap":700,"maxSpans":500000,"node":{"cpus":2},"probeStartupDelay":200,"queryPort":9411,"resources":{"limits":{"cpu":"300m","memory":"900Mi"},"requests":{"cpu":"150m","memory":"900Mi"}},"tag":"2.14.2"}},"version":""} - - config: |- - policy: enabled - alwaysInjectSelector: - [] - neverInjectSelector: - [] - template: | - rewriteAppHTTPProbe: {{ valueOrDefault .Values.sidecarInjectorWebhook.rewriteAppHTTPProbe false }} - {{- if or (not .Values.istio_cni.enabled) .Values.global.proxy.enableCoreDump }} - initContainers: - {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} - {{- if not .Values.istio_cni.enabled }} - - name: istio-init - {{- if contains "/" .Values.global.proxy_init.image }} - image: "{{ .Values.global.proxy_init.image }}" - {{- else }} - image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" - {{- end }} - command: - - istio-iptables - - "-p" - - 15001 - - "-z" - - "15006" - - "-u" - - 1337 - - "-m" - - "{{ annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode }}" - - "-i" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeOutboundIPRanges` .Values.global.proxy.includeIPRanges }}" - - "-x" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundIPRanges` .Values.global.proxy.excludeIPRanges }}" - - "-b" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` `*` }}" - - "-d" - - "{{ excludeInboundPort (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) (annotation .ObjectMeta `traffic.sidecar.istio.io/excludeInboundPorts` .Values.global.proxy.excludeInboundPorts) }}" - {{ if or (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/excludeOutboundPorts`) (ne (valueOrDefault .Values.global.proxy.excludeOutboundPorts "") "") -}} - - "-o" - - "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/excludeOutboundPorts` .Values.global.proxy.excludeOutboundPorts }}" - {{ end -}} - {{ if (isset .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces`) -}} - - "-k" - - "{{ index .ObjectMeta.Annotations `traffic.sidecar.istio.io/kubevirtInterfaces` }}" - {{ end -}} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" - {{- if .Values.global.proxy_init.resources }} - resources: - {{ toYaml .Values.global.proxy_init.resources | indent 4 }} - {{- else }} - resources: {} - {{- end }} - securityContext: - runAsUser: 0 - runAsNonRoot: false - capabilities: - add: - - NET_ADMIN - {{- if .Values.global.proxy.privileged }} - privileged: true - {{- end }} - restartPolicy: Always - {{- end }} - {{ end -}} - {{- if eq .Values.global.proxy.enableCoreDump true }} - - name: enable-core-dump - args: - - -c - - sysctl -w kernel.core_pattern=/var/lib/istio/core.proxy && ulimit -c unlimited - command: - - /bin/sh - {{- if contains "/" .Values.global.proxy_init.image }} - image: "{{ .Values.global.proxy_init.image }}" - {{- else }} - image: "{{ .Values.global.hub }}/{{ .Values.global.proxy_init.image }}:{{ .Values.global.tag }}" - {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" - resources: {} - securityContext: - runAsUser: 0 - runAsNonRoot: false - privileged: true - {{ end }} - {{- end }} - containers: - - name: istio-proxy - {{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image) }} - image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image }}" - {{- else }} - image: "{{ .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}" - {{- end }} - ports: - - containerPort: 15090 - protocol: TCP - name: http-envoy-prom - args: - - proxy - - sidecar - - --domain - - $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }} - - --configPath - - "/etc/istio/proxy" - - --binaryPath - - "/usr/local/bin/envoy" - - --serviceCluster - {{ if ne "" (index .ObjectMeta.Labels "app") -}} - - "{{ index .ObjectMeta.Labels `app` }}.$(POD_NAMESPACE)" - {{ else -}} - - "{{ valueOrDefault .DeploymentMeta.Name `istio-proxy` }}.{{ valueOrDefault .DeploymentMeta.Namespace `default` }}" - {{ end -}} - - --drainDuration - - "{{ formatDuration .ProxyConfig.DrainDuration }}" - - --parentShutdownDuration - - "{{ formatDuration .ProxyConfig.ParentShutdownDuration }}" - - --discoveryAddress - - "{{ annotation .ObjectMeta `sidecar.istio.io/discoveryAddress` .ProxyConfig.DiscoveryAddress }}" - {{- if eq .Values.global.proxy.tracer "lightstep" }} - - --lightstepAddress - - "{{ .ProxyConfig.GetTracing.GetLightstep.GetAddress }}" - - --lightstepAccessToken - - "{{ .ProxyConfig.GetTracing.GetLightstep.GetAccessToken }}" - - --lightstepSecure={{ .ProxyConfig.GetTracing.GetLightstep.GetSecure }} - - --lightstepCacertPath - - "{{ .ProxyConfig.GetTracing.GetLightstep.GetCacertPath }}" - {{- else if eq .Values.global.proxy.tracer "zipkin" }} - - --zipkinAddress - - "{{ .ProxyConfig.GetTracing.GetZipkin.GetAddress }}" - {{- else if eq .Values.global.proxy.tracer "datadog" }} - - --datadogAgentAddress - - "{{ .ProxyConfig.GetTracing.GetDatadog.GetAddress }}" - {{- end }} - - --proxyLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/logLevel` .Values.global.proxy.logLevel}} - - --proxyComponentLogLevel={{ annotation .ObjectMeta `sidecar.istio.io/componentLogLevel` .Values.global.proxy.componentLogLevel}} - - --connectTimeout - - "{{ formatDuration .ProxyConfig.ConnectTimeout }}" - {{- if .Values.global.proxy.envoyStatsd.enabled }} - - --statsdUdpAddress - - "{{ .ProxyConfig.StatsdUdpAddress }}" - {{- end }} - {{- if .Values.global.proxy.envoyMetricsService.enabled }} - - --envoyMetricsServiceAddress - - "{{ .ProxyConfig.GetEnvoyMetricsService.GetAddress }}" - {{- end }} - {{- if .Values.global.proxy.envoyAccessLogService.enabled }} - - --envoyAccessLogServiceAddress - - "{{ .ProxyConfig.GetEnvoyAccessLogService.GetAddress }}" - {{- end }} - - --proxyAdminPort - - "{{ .ProxyConfig.ProxyAdminPort }}" - {{ if gt .ProxyConfig.Concurrency 0 -}} - - --concurrency - - "{{ .ProxyConfig.Concurrency }}" - {{ end -}} - {{- if .Values.global.controlPlaneSecurityEnabled }} - - --controlPlaneAuthPolicy - - MUTUAL_TLS - {{- else }} - - --controlPlaneAuthPolicy - - NONE - {{- end }} - - --dnsRefreshRate - - {{ valueOrDefault .Values.global.proxy.dnsRefreshRate "300s" }} - {{- if (ne (annotation .ObjectMeta "status.sidecar.istio.io/port" .Values.global.proxy.statusPort) "0") }} - - --statusPort - - "{{ annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort }}" - - --applicationPorts - - "{{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/applicationPorts` (applicationPorts .Spec.Containers) }}" - {{- end }} - {{- if .Values.global.trustDomain }} - - --trust-domain={{ .Values.global.trustDomain }} - {{- end }} - {{- if .Values.global.logAsJson }} - - --log_as_json - {{- end }} - {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} - - --templateFile=/etc/istio/custom-bootstrap/envoy_bootstrap.json - {{- end }} - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: INSTANCE_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - {{- if eq .Values.global.proxy.tracer "datadog" }} - {{- if isset .ObjectMeta.Annotations `apm.datadoghq.com/env` }} - {{- range $key, $value := fromJSON (index .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} - - name: {{ $key }} - value: "{{ $value }}" - {{- end }} - {{- end }} - {{- end }} - - name: ISTIO_META_POD_PORTS - value: |- - [ - {{- range $index1, $c := .Spec.Containers }} - {{- range $index2, $p := $c.Ports }} - {{if or (ne $index1 0) (ne $index2 0)}},{{end}}{{ structToJSON $p }} - {{- end}} - {{- end}} - ] - - name: ISTIO_META_CLUSTER_ID - value: "{{ valueOrDefault .Values.global.multiCluster.clusterName `Kubernetes` }}" - - name: ISTIO_META_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: ISTIO_META_CONFIG_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: SDS_ENABLED - value: "{{ .Values.global.sds.enabled }}" - - name: ISTIO_META_INTERCEPTION_MODE - value: "{{ or (index .ObjectMeta.Annotations `sidecar.istio.io/interceptionMode`) .ProxyConfig.InterceptionMode.String }}" - - name: ISTIO_META_INCLUDE_INBOUND_PORTS - value: "{{ annotation .ObjectMeta `traffic.sidecar.istio.io/includeInboundPorts` (applicationPorts .Spec.Containers) }}" - {{- if .Values.global.network }} - - name: ISTIO_META_NETWORK - value: "{{ .Values.global.network }}" - {{- end }} - {{ if .ObjectMeta.Annotations }} - - name: ISTIO_METAJSON_ANNOTATIONS - value: | - {{ toJSON .ObjectMeta.Annotations }} - {{ end }} - {{ if .ObjectMeta.Labels }} - - name: ISTIO_METAJSON_LABELS - value: | - {{ toJSON .ObjectMeta.Labels }} - {{ end }} - {{- if .DeploymentMeta.Name }} - - name: ISTIO_META_WORKLOAD_NAME - value: {{ .DeploymentMeta.Name }} - {{ end }} - {{- if and .TypeMeta.APIVersion .DeploymentMeta.Name }} - - name: ISTIO_META_OWNER - value: kubernetes://api/{{ .TypeMeta.APIVersion }}/namespaces/{{ valueOrDefault .DeploymentMeta.Namespace `default` }}/{{ toLower .TypeMeta.Kind}}s/{{ .DeploymentMeta.Name }} - {{- end}} - {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} - - name: ISTIO_BOOTSTRAP_OVERRIDE - value: "/etc/istio/custom-bootstrap/custom_bootstrap.json" - {{- end }} - {{- if .Values.global.sds.customTokenDirectory }} - - name: ISTIO_META_SDS_TOKEN_PATH - value: "{{ .Values.global.sds.customTokenDirectory -}}/sdstoken" - {{- end }} - {{- if .Values.global.meshID }} - - name: ISTIO_META_MESH_ID - value: "{{ .Values.global.meshID }}" - {{- else if .Values.global.trustDomain }} - - name: ISTIO_META_MESH_ID - value: "{{ .Values.global.trustDomain }}" - {{- end }} - {{- if and (eq .Values.global.proxy.tracer "datadog") (isset .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} - {{- range $key, $value := fromJSON (index .ObjectMeta.Annotations `apm.datadoghq.com/env`) }} - - name: {{ $key }} - value: "{{ $value }}" - {{- end }} - {{- end }} - imagePullPolicy: "{{ valueOrDefault .Values.global.imagePullPolicy `Always` }}" - {{ if ne (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) `0` }} - readinessProbe: - httpGet: - path: /healthz/ready - port: {{ annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort }} - initialDelaySeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/initialDelaySeconds` .Values.global.proxy.readinessInitialDelaySeconds }} - periodSeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/periodSeconds` .Values.global.proxy.readinessPeriodSeconds }} - failureThreshold: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/failureThreshold` .Values.global.proxy.readinessFailureThreshold }} - {{ end -}} - securityContext: - {{- if .Values.global.proxy.privileged }} - privileged: true - {{- end }} - {{- if ne .Values.global.proxy.enableCoreDump true }} - readOnlyRootFilesystem: true - {{- end }} - {{ if eq (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `TPROXY` -}} - capabilities: - add: - - NET_ADMIN - runAsGroup: 1337 - {{ else -}} - {{ if .Values.global.sds.enabled }} - runAsGroup: 1337 - {{- end }} - runAsUser: 1337 - {{- end }} - resources: - {{ if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} - requests: - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) -}} - cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU` }}" - {{ end}} - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}} - memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory` }}" - {{ end }} - {{ else -}} - {{- if .Values.global.proxy.resources }} - {{ toYaml .Values.global.proxy.resources | indent 4 }} - {{- end }} - {{ end -}} - volumeMounts: - {{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} - - mountPath: /etc/istio/custom-bootstrap - name: custom-bootstrap-volume - {{- end }} - - mountPath: /etc/istio/proxy - name: istio-envoy - {{- if .Values.global.sds.enabled }} - - mountPath: /var/run/sds - name: sds-uds-path - readOnly: true - - mountPath: /var/run/secrets/tokens - name: istio-token - {{- if .Values.global.sds.customTokenDirectory }} - - mountPath: "{{ .Values.global.sds.customTokenDirectory -}}" - name: custom-sds-token - readOnly: true - {{- end }} - {{- else }} - - mountPath: /etc/certs/ - name: istio-certs - readOnly: true - {{- end }} - {{- if and (eq .Values.global.proxy.tracer "lightstep") .Values.global.tracer.lightstep.cacertPath }} - - mountPath: {{ directory .ProxyConfig.GetTracing.GetLightstep.GetCacertPath }} - name: lightstep-certs - readOnly: true - {{- end }} - {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount` }} - {{ range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount`) }} - - name: "{{ $index }}" - {{ toYaml $value | indent 4 }} - {{ end }} - {{- end }} - volumes: - {{- if (isset .ObjectMeta.Annotations `sidecar.istio.io/bootstrapOverride`) }} - - name: custom-bootstrap-volume - configMap: - name: {{ annotation .ObjectMeta `sidecar.istio.io/bootstrapOverride` "" }} - {{- end }} - - emptyDir: - medium: Memory - name: istio-envoy - {{- if .Values.global.sds.enabled }} - - name: sds-uds-path - hostPath: - path: /var/run/sds - - name: istio-token - projected: - sources: - - serviceAccountToken: - path: istio-token - expirationSeconds: 43200 - audience: {{ .Values.global.sds.token.aud }} - {{- if .Values.global.sds.customTokenDirectory }} - - name: custom-sds-token - secret: - secretName: sdstokensecret - {{- end }} - {{- else }} - - name: istio-certs - secret: - optional: true - {{ if eq .Spec.ServiceAccountName "" }} - secretName: istio.default - {{ else -}} - secretName: {{ printf "istio.%s" .Spec.ServiceAccountName }} - {{ end -}} - {{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolume` }} - {{range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolume`) }} - - name: "{{ $index }}" - {{ toYaml $value | indent 2 }} - {{ end }} - {{ end }} - {{- end }} - {{- if and (eq .Values.global.proxy.tracer "lightstep") .Values.global.tracer.lightstep.cacertPath }} - - name: lightstep-certs - secret: - optional: true - secretName: lightstep.cacert - {{- end }} - {{- if .Values.global.podDNSSearchNamespaces }} - dnsConfig: - searches: - {{- range .Values.global.podDNSSearchNamespaces }} - - {{ render . }} - {{- end }} - {{- end }} - injectedAnnotations: ---- - -# Resources for Kiali component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kiali - labels: - app: kiali - release: istio -rules: - - apiGroups: [""] - resources: - - configmaps - - endpoints - - namespaces - - nodes - - pods - - pods/log - - replicationcontrollers - - services - verbs: - - get - - list - - watch - - apiGroups: ["extensions", "apps"] - resources: - - deployments - - replicasets - - statefulsets - verbs: - - get - - list - - watch - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: - - get - - list - - watch - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: - - get - - list - - watch - - apiGroups: - - config.istio.io - - networking.istio.io - - authentication.istio.io - - rbac.istio.io - - security.istio.io - resources: ["*"] - verbs: - - create - - delete - - get - - list - - patch - - watch - - apiGroups: ["monitoring.kiali.io"] - resources: - - monitoringdashboards - verbs: - - get - - list ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: kiali-viewer - labels: - app: kiali - release: istio -rules: - - apiGroups: [""] - resources: - - configmaps - - endpoints - - namespaces - - nodes - - pods - - pods/log - - replicationcontrollers - - services - verbs: - - get - - list - - watch - - apiGroups: ["extensions", "apps"] - resources: - - deployments - - replicasets - - statefulsets - verbs: - - get - - list - - watch - - apiGroups: ["autoscaling"] - resources: - - horizontalpodautoscalers - verbs: - - get - - list - - watch - - apiGroups: ["batch"] - resources: - - cronjobs - - jobs - verbs: - - get - - list - - watch - - apiGroups: - - config.istio.io - - networking.istio.io - - authentication.istio.io - - rbac.istio.io - - security.istio.io - resources: ["*"] - verbs: - - get - - list - - watch - - apiGroups: ["monitoring.kiali.io"] - resources: - - monitoringdashboards - verbs: - - get - - list ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kiali - labels: - app: kiali - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kiali -subjects: - - kind: ServiceAccount - name: kiali-service-account - namespace: istio-system ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: kiali - namespace: istio-system - labels: - app: kiali - release: istio -data: - config.yaml: | - istio_component_namespaces: - grafana: istio-system - tracing: istio-system - pilot: istio-system - prometheus: istio-system - istio_namespace: istio-system - deployment: - accessible_namespaces: ['**'] - server: - port: 20001 - web_root: /kiali - external_services: - istio: - url_service_version: http://istio-pilot.istio-system:8080/version - tracing: - url: - grafana: - url: - prometheus: - url: http://prometheus.istio-system:9090 ---- - - -apiVersion: v1 -kind: Secret -metadata: - name: kiali - namespace: istio-system - labels: - app: kiali - release: istio -type: Opaque -data: - username: YWRtaW4= # admin - passphrase: YWRtaW4= # admin ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kiali - namespace: istio-system - labels: - app: kiali - release: istio -spec: - replicas: 1 - selector: - matchLabels: - app: kiali - template: - metadata: - name: kiali - labels: - app: kiali - release: istio - annotations: - sidecar.istio.io/inject: "false" - scheduler.alpha.kubernetes.io/critical-pod: "" - prometheus.io/scrape: "true" - prometheus.io/port: "9090" - kiali.io/runtimes: go,kiali - spec: - serviceAccountName: kiali-service-account - containers: - - image: "quay.io/kiali/kiali:v1.9" - imagePullPolicy: IfNotPresent - name: kiali - command: - - "/opt/kiali/kiali" - - "-config" - - "/kiali-configuration/config.yaml" - - "-v" - - "3" - readinessProbe: - httpGet: - path: /kiali/healthz - port: 20001 - scheme: 'HTTP' - initialDelaySeconds: 5 - periodSeconds: 30 - livenessProbe: - httpGet: - path: /kiali/healthz - port: 20001 - scheme: 'HTTP' - initialDelaySeconds: 5 - periodSeconds: 30 - env: - - name: ACTIVE_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: kiali-configuration - mountPath: "/kiali-configuration" - - name: kiali-cert - mountPath: "/kiali-cert" - - name: kiali-secret - mountPath: "/kiali-secret" - resources: - requests: - cpu: 10m - - volumes: - - name: kiali-configuration - configMap: - name: kiali - - name: kiali-cert - secret: - secretName: istio.kiali-service-account - optional: true - - name: kiali-secret - secret: - secretName: kiali - optional: true - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x ---- - - -apiVersion: v1 -kind: Service -metadata: - name: kiali - namespace: istio-system - labels: - app: kiali - release: istio -spec: - ports: - - name: http-kiali - protocol: TCP - port: 20001 - selector: - app: kiali ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kiali-service-account - namespace: istio-system - labels: - app: kiali - release: istio ---- - -# NodeAgent component is disabled. - -# Resources for Pilot component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-pilot-istio-system - labels: - app: pilot - release: istio -rules: -- apiGroups: ["config.istio.io"] - resources: ["*"] - verbs: ["*"] -- apiGroups: ["rbac.istio.io"] - resources: ["*"] - verbs: ["get", "watch", "list"] -- apiGroups: ["security.istio.io"] - resources: ["*"] - verbs: ["get", "watch", "list"] -- apiGroups: ["networking.istio.io"] - resources: ["*"] - verbs: ["*"] -- apiGroups: ["authentication.istio.io"] - resources: ["*"] - verbs: ["*"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["*"] -- apiGroups: ["extensions"] - resources: ["ingresses", "ingresses/status"] - verbs: ["*"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["create", "get", "list", "watch", "update"] -- apiGroups: [""] - resources: ["endpoints", "pods", "services", "namespaces", "nodes", "secrets"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["create", "get", "watch", "list", "update", "delete"] -- apiGroups: ["certificates.k8s.io"] - resources: - - "certificatesigningrequests" - - "certificatesigningrequests/approval" - - "certificatesigningrequests/status" - verbs: ["update", "create", "get", "delete"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: istio-pilot-istio-system - labels: - app: pilot - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: istio-pilot-istio-system -subjects: - - kind: ServiceAccount - name: istio-pilot-service-account - namespace: istio-system ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: istio-system - name: pilot-envoy-config - labels: - release: istio -data: - envoy.yaml.tmpl: |- - admin: - access_log_path: /dev/null - address: - socket_address: - address: 127.0.0.1 - port_value: 15000 - - static_resources: - clusters: - - name: in.15010 - http2_protocol_options: {} - connect_timeout: 1.000s - - hosts: - - socket_address: - address: 127.0.0.1 - port_value: 15010 - - circuit_breakers: - thresholds: - - max_connections: 100000 - max_pending_requests: 100000 - max_requests: 100000 - max_retries: 3 - - # TODO: telemetry using EDS - # TODO: other pilots using EDS, load balancing - # TODO: galley using EDS - - - name: out.galley.15019 - http2_protocol_options: {} - connect_timeout: 1.000s - type: STRICT_DNS - - circuit_breakers: - thresholds: - - max_connections: 100000 - max_pending_requests: 100000 - max_requests: 100000 - max_retries: 3 - - tls_context: - common_tls_context: - tls_certificates: - - certificate_chain: - filename: /etc/certs/cert-chain.pem - private_key: - filename: /etc/certs/key.pem - validation_context: - trusted_ca: - filename: /etc/certs/root-cert.pem - verify_subject_alt_name: - - spiffe://cluster.local/ns/istio-system/sa/istio-galley-service-account - - hosts: - - socket_address: - address: istio-galley.istio-system - port_value: 15019 - - - listeners: - - name: "in.15011" - address: - socket_address: - address: 0.0.0.0 - port_value: 15011 - filter_chains: - - filters: - - name: envoy.http_connection_manager - #typed_config - #"@type": "type.googleapis.com/", - config: - codec_type: HTTP2 - stat_prefix: "15011" - http2_protocol_options: - max_concurrent_streams: 1073741824 - - access_log: - - name: envoy.file_access_log - config: - path: /dev/stdout - - http_filters: - - name: envoy.router - - route_config: - name: "15011" - - virtual_hosts: - - name: istio-pilot - - domains: - - '*' - - routes: - - match: - prefix: / - route: - cluster: in.15010 - timeout: 0.000s - decorator: - operation: xDS - - tls_context: - require_client_certificate: true - common_tls_context: - validation_context: - trusted_ca: - filename: /etc/certs/root-cert.pem - - alpn_protocols: - - h2 - - tls_certificates: - - certificate_chain: - filename: /etc/certs/cert-chain.pem - private_key: - filename: /etc/certs/key.pem - - - # Manual 'whitebox' mode - - name: "local.15019" - address: - socket_address: - address: 127.0.0.1 - port_value: 15019 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - codec_type: HTTP2 - stat_prefix: "15019" - http2_protocol_options: - max_concurrent_streams: 1073741824 - - access_log: - - name: envoy.file_access_log - config: - path: /dev/stdout - - http_filters: - - name: envoy.router - - route_config: - name: "15019" - - virtual_hosts: - - name: istio-galley - - domains: - - '*' - - routes: - - match: - prefix: / - route: - cluster: out.galley.15019 - timeout: 0.000s ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: istio - namespace: istio-system - labels: - release: istio -data: - - meshNetworks: |- - # Network config - networks: {} - - values.yaml: |- - appNamespaces: [] - autoscaleEnabled: false - autoscaleMax: 5 - autoscaleMin: 1 - configMap: true - configNamespace: istio-config - cpu: - targetAverageUtilization: 80 - enableProtocolSniffingForInbound: false - enableProtocolSniffingForOutbound: true - enabled: true - env: {} - image: pilot - ingress: - ingressClass: istio - ingressControllerMode: "OFF" - ingressService: istio-ingressgateway - keepaliveMaxServerConnectionAge: 30m - meshNetworks: - networks: {} - namespace: istio-system - nodeSelector: {} - plugins: [] - podAnnotations: {} - podAntiAffinityLabelSelector: [] - podAntiAffinityTermLabelSelector: [] - policy: - enabled: false - replicaCount: 1 - resources: - requests: - cpu: 500m - memory: 2048Mi - rollingMaxSurge: 100% - rollingMaxUnavailable: 25% - tolerations: [] - traceSampling: 1 - useMCP: true - - mesh: |- - # Set enableTracing to false to disable request tracing. - enableTracing: true - - # Set accessLogFile to empty string to disable access log. - accessLogFile: "/dev/stdout" - - enableEnvoyAccessLogService: false - mixerCheckServer: istio-policy.istio-system.svc.cluster.local:9091 - mixerReportServer: istio-telemetry.istio-system.svc.cluster.local:9091 - # policyCheckFailOpen allows traffic in cases when the mixer policy service cannot be reached. - # Default is false which means the traffic is denied when the client is unable to connect to Mixer. - policyCheckFailOpen: false - # reportBatchMaxEntries is the number of requests that are batched before telemetry data is sent to the mixer server - reportBatchMaxEntries: 100 - # reportBatchMaxTime is the max waiting time before the telemetry data of a request is sent to the mixer server - reportBatchMaxTime: 1s - disableMixerHttpReports: false - - disablePolicyChecks: true - - - # This is the k8s ingress service name, update if you used a different name - ingressService: "istio-ingressgateway" - ingressControllerMode: "OFF" - ingressClass: "istio" - - # The trust domain corresponds to the trust root of a system. - # Refer to https://github.com/spiffe/spiffe/blob/master/standards/SPIFFE-ID.md#21-trust-domain - trustDomain: "cluster.local" - - # The trust domain aliases represent the aliases of trust_domain. - # For example, if we have - # trustDomain: td1 - # trustDomainAliases: [“td2”, "td3"] - # Any service with the identity "td1/ns/foo/sa/a-service-account", "td2/ns/foo/sa/a-service-account", - # or "td3/ns/foo/sa/a-service-account" will be treated the same in the Istio mesh. - trustDomainAliases: - - # Set expected values when SDS is disabled - # Unix Domain Socket through which envoy communicates with NodeAgent SDS to get - # key/cert for mTLS. Use secret-mount files instead of SDS if set to empty. - sdsUdsPath: "" - - # This flag is used by secret discovery service(SDS). - # If set to true(prerequisite: https://kubernetes.io/docs/concepts/storage/volumes/#projected), Istio will inject volumes mount - # for k8s service account JWT, so that K8s API server mounts k8s service account JWT to envoy container, which - # will be used to generate key/cert eventually. This isn't supported for non-k8s case. - enableSdsTokenMount: false - - # This flag is used by secret discovery service(SDS). - # If set to true, envoy will fetch normal k8s service account JWT from '/var/run/secrets/kubernetes.io/serviceaccount/token' - # (https://kubernetes.io/docs/tasks/access-application-cluster/access-cluster/#accessing-the-api-from-a-pod) - # and pass to sds server, which will be used to request key/cert eventually. - # this flag is ignored if enableSdsTokenMount is set. - # This isn't supported for non-k8s case. - sdsUseK8sSaJwt: false - - # If true, automatically configure client side mTLS settings to match the corresponding service's - # server side mTLS authentication policy, when destination rule for that service does not specify - # TLS settings. - enableAutoMtls: false - config_sources: - - address: istio-galley.istio-system:9901 - - outboundTrafficPolicy: - mode: ALLOW_ANY - localityLbSetting: - enabled: true - - # Configures DNS certificates provisioned through Chiron linked into Pilot. - # The DNS certificate provisioning is enabled by default now so it get tested. - # TODO (lei-tang): we'll decide whether enable it by default or not before Istio 1.4 Release. - certificates: - [] - - defaultConfig: - # - # TCP connection timeout between Envoy & the application, and between Envoys. - connectTimeout: 10s - # - ### ADVANCED SETTINGS ############# - # Where should envoy's configuration be stored in the istio-proxy container - configPath: "/etc/istio/proxy" - # The pseudo service name used for Envoy. - serviceCluster: istio-proxy - # These settings that determine how long an old Envoy - # process should be kept alive after an occasional reload. - drainDuration: 45s - parentShutdownDuration: 1m0s - # - # Port where Envoy listens (on local host) for admin commands - # You can exec into the istio-proxy container in a pod and - # curl the admin port (curl http://localhost:15000/) to obtain - # diagnostic information from Envoy. See - # https://lyft.github.io/envoy/docs/operations/admin.html - # for more details - proxyAdminPort: 15000 - # - # Set concurrency to a specific number to control the number of Proxy worker threads. - # If set to 0 (default), then start worker thread for each CPU thread/core. - concurrency: 2 - # - tracing: - zipkin: - # Address of the Zipkin collector - address: zipkin.istio-system:9411 - # - # Mutual TLS authentication between sidecars and istio control plane. - controlPlaneAuthPolicy: NONE - # - # Address where istio Pilot service is running - discoveryAddress: istio-pilot.istio-system:15010 ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: pilot - istio: pilot - release: istio - name: istio-pilot - namespace: istio-system -spec: - replicas: 1 - selector: - matchLabels: - istio: pilot - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: pilot - chart: pilot - heritage: Tiller - istio: pilot - release: istio - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - args: - - discovery - - --monitoringAddr=:15014 - - --log_output_level=default:info - - --domain - - cluster.local - - --secureGrpcAddr - - "" - - --trust-domain=cluster.local - - --keepaliveMaxServerConnectionAge - - 30m - env: - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: GODEBUG - value: gctrace=1 - - name: PILOT_TRACE_SAMPLING - value: "100" - - name: CONFIG_NAMESPACE - value: istio-config - - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_OUTBOUND - value: "true" - - name: PILOT_ENABLE_PROTOCOL_SNIFFING_FOR_INBOUND - value: "false" - image: docker.io/istio/pilot:1.4.2 - imagePullPolicy: IfNotPresent - name: discovery - ports: - - containerPort: 8080 - - containerPort: 15010 - readinessProbe: - httpGet: - path: /ready - port: 8080 - initialDelaySeconds: 5 - periodSeconds: 30 - timeoutSeconds: 5 - resources: - requests: - cpu: 10m - memory: 100Mi - volumeMounts: - - mountPath: /etc/istio/config - name: config-volume - serviceAccountName: istio-pilot-service-account - volumes: - - configMap: - name: istio - name: config-volume - - configMap: - name: pilot-envoy-config - name: pilot-envoy-config - ---- - - -apiVersion: "authentication.istio.io/v1alpha1" -kind: "MeshPolicy" -metadata: - name: "default" - labels: - release: istio -spec: - peers: - - mtls: - mode: PERMISSIVE ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: istio-pilot - namespace: istio-system - labels: - app: pilot - release: istio - istio: pilot -spec: - minAvailable: 1 - selector: - matchLabels: - app: pilot - release: istio - istio: pilot ---- - - -apiVersion: v1 -kind: Service -metadata: - name: istio-pilot - namespace: istio-system - labels: - app: pilot - release: istio - istio: pilot -spec: - ports: - - port: 15010 - name: grpc-xds # direct - - port: 15011 - name: https-xds # mTLS - - port: 8080 - name: http-legacy-discovery # direct - - port: 15014 - name: http-monitoring - selector: - istio: pilot ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-pilot-service-account - namespace: istio-system - labels: - app: pilot - release: istio ---- - -# Resources for Policy component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-policy - labels: - release: istio - app: istio-policy -rules: -- apiGroups: ["config.istio.io"] # istio CRD watcher - resources: ["*"] - verbs: ["create", "get", "list", "watch", "patch"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["configmaps", "endpoints", "pods", "services", "namespaces", "secrets", "replicationcontrollers"] - verbs: ["get", "list", "watch"] -- apiGroups: ["extensions", "apps"] - resources: ["replicasets"] - verbs: ["get", "list", "watch"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: istio-policy-admin-role-binding-istio-system - labels: - app: istio-policy - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: istio-policy -subjects: - - kind: ServiceAccount - name: istio-policy-service-account - namespace: istio-system ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: istio-policy - namespace: istio-system - labels: - app: istio-policy - release: istio -spec: - host: istio-policy.istio-system.svc.cluster.local - trafficPolicy: - portLevelSettings: - - port: - number: 15004 # grpc-mixer-mtls - tls: - mode: ISTIO_MUTUAL - - port: - number: 9091 # grpc-mixer - tls: - mode: DISABLE - connectionPool: - http: - http2MaxRequests: 10000 - maxRequestsPerConnection: 10000 ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: istio-policy - istio: mixer - release: istio - name: istio-policy - namespace: istio-system -spec: - replicas: 1 - selector: - matchLabels: - istio: mixer - istio-mixer-type: policy - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: policy - istio: mixer - istio-mixer-type: policy - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - args: - - --monitoringPort=15014 - - --address - - tcp://0.0.0.0:9091 - - --log_output_level=default:info - - --configStoreURL=mcp://istio-galley.istio-system.svc:9901 - - --configDefaultNamespace=istio-system - - --useAdapterCRDs=false - - --useTemplateCRDs=false - - --trace_zipkin_url=http://zipkin.istio-system:9411/api/v1/spans - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - image: docker.io/istio/mixer:1.4.2 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /version - port: 15014 - initialDelaySeconds: 5 - periodSeconds: 5 - name: mixer - ports: - - containerPort: 9091 - - containerPort: 15014 - - containerPort: 42422 - resources: - requests: - cpu: 10m - memory: 100Mi - volumeMounts: - - mountPath: /etc/certs - name: istio-certs - readOnly: true - - mountPath: /sock - name: uds-socket - - mountPath: /var/run/secrets/istio.io/policy/adapter - name: policy-adapter-secret - readOnly: true - serviceAccountName: istio-policy-service-account - volumes: - - name: istio-certs - secret: - optional: true - secretName: istio.istio-policy-service-account - - emptyDir: {} - name: uds-socket - - name: policy-adapter-secret - secret: - optional: true - secretName: policy-adapter-secret - ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: istio-policy - namespace: istio-system - labels: - app: policy - release: istio - istio: mixer - istio-mixer-type: policy -spec: - minAvailable: 1 - selector: - matchLabels: - app: policy - release: istio - istio: mixer - istio-mixer-type: policy ---- - - -apiVersion: v1 -kind: Service -metadata: - name: istio-policy - namespace: istio-system - labels: - app: mixer - istio: mixer - release: istio -spec: - ports: - - name: grpc-mixer - port: 9091 - - name: grpc-mixer-mtls - port: 15004 - - name: http-policy-monitoring - port: 15014 - selector: - istio: mixer - istio-mixer-type: policy ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-policy-service-account - namespace: istio-system - labels: - app: istio-policy - release: istio ---- - -# Resources for Prometheus component - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: prometheus-istio-system - labels: - app: prometheus - release: istio -rules: -- apiGroups: [""] - resources: - - nodes - - services - - endpoints - - pods - - nodes/proxy - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: - - configmaps - verbs: ["get"] -- nonResourceURLs: ["/metrics"] - verbs: ["get"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: prometheus-istio-system - labels: - app: prometheus - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: prometheus-istio-system -subjects: -- kind: ServiceAccount - name: prometheus - namespace: istio-system ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus - namespace: istio-system - labels: - app: prometheus - release: istio -data: - prometheus.yml: |- - global: - scrape_interval: 15s - scrape_configs: - - # Mixer scrapping. Defaults to Prometheus and mixer on same namespace. - # - - job_name: 'istio-mesh' - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - istio-system - relabel_configs: - - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - action: keep - regex: istio-telemetry;prometheus - - # Scrape config for envoy stats - - job_name: 'envoy-stats' - metrics_path: /stats/prometheus - kubernetes_sd_configs: - - role: pod - - relabel_configs: - - source_labels: [__meta_kubernetes_pod_container_port_name] - action: keep - regex: '.*-envoy-prom' - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:15090 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - action: replace - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: pod_name - - - job_name: 'istio-policy' - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - istio-system - - - relabel_configs: - - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - action: keep - regex: istio-policy;http-policy-monitoring - - - job_name: 'istio-telemetry' - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - istio-system - - relabel_configs: - - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - action: keep - regex: istio-telemetry;http-monitoring - - - job_name: 'pilot' - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - istio-system - - relabel_configs: - - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - action: keep - regex: istio-pilot;http-monitoring - - - job_name: 'galley' - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - istio-system - - relabel_configs: - - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - action: keep - regex: istio-galley;http-monitoring - - - job_name: 'citadel' - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - istio-system - - relabel_configs: - - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - action: keep - regex: istio-citadel;http-monitoring - - # scrape config for API servers - - job_name: 'kubernetes-apiservers' - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - default - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - relabel_configs: - - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] - action: keep - regex: kubernetes;https - - # scrape config for nodes (kubelet) - - job_name: 'kubernetes-nodes' - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics - - # Scrape config for Kubelet cAdvisor. - # - # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics - # (those whose names begin with 'container_') have been removed from the - # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to - # retrieve those metrics. - # - # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor - # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" - # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with - # the --cadvisor-port=0 Kubelet flag). - # - # This job is not necessary and should be removed in Kubernetes 1.6 and - # earlier versions, or it will cause the metrics to be scraped twice. - - job_name: 'kubernetes-cadvisor' - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - relabel_configs: - - action: labelmap - regex: __meta_kubernetes_node_label_(.+) - - target_label: __address__ - replacement: kubernetes.default.svc:443 - - source_labels: [__meta_kubernetes_node_name] - regex: (.+) - target_label: __metrics_path__ - replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor - - # scrape config for service endpoints. - - job_name: 'kubernetes-service-endpoints' - kubernetes_sd_configs: - - role: endpoints - relabel_configs: - - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] - action: replace - target_label: __scheme__ - regex: (https?) - - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] - action: replace - target_label: __address__ - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - - action: labelmap - regex: __meta_kubernetes_service_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - action: replace - target_label: kubernetes_namespace - - source_labels: [__meta_kubernetes_service_name] - action: replace - target_label: kubernetes_name - - - job_name: 'kubernetes-pods' - kubernetes_sd_configs: - - role: pod - relabel_configs: # If first two labels are present, pod should be scraped by the istio-secure job. - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status] - action: drop - regex: (.+) - - source_labels: [__meta_kubernetes_pod_annotation_istio_mtls] - action: drop - regex: (true) - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - action: replace - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: pod_name - - job_name: 'kubernetes-pods-istio-secure' - scheme: https - tls_config: - ca_file: /etc/istio-certs/root-cert.pem - cert_file: /etc/istio-certs/cert-chain.pem - key_file: /etc/istio-certs/key.pem - insecure_skip_verify: true # prometheus does not support secure naming. - kubernetes_sd_configs: - - role: pod - relabel_configs: - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] - action: keep - regex: true - # sidecar status annotation is added by sidecar injector and - # istio_workload_mtls_ability can be specifically placed on a pod to indicate its ability to receive mtls traffic. - - source_labels: [__meta_kubernetes_pod_annotation_sidecar_istio_io_status, __meta_kubernetes_pod_annotation_istio_mtls] - action: keep - regex: (([^;]+);([^;]*))|(([^;]*);(true)) - - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: [__address__] # Only keep address that is host:port - action: keep # otherwise an extra target with ':443' is added for https scheme - regex: ([^:]+):(\d+) - - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] - action: replace - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: [__meta_kubernetes_namespace] - action: replace - target_label: namespace - - source_labels: [__meta_kubernetes_pod_name] - action: replace - target_label: pod_name ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: prometheus - namespace: istio-system - labels: - app: prometheus - release: istio -spec: - replicas: 1 - selector: - matchLabels: - app: prometheus - template: - metadata: - labels: - app: prometheus - release: istio - annotations: - sidecar.istio.io/inject: "false" - spec: - serviceAccountName: prometheus - containers: - - name: prometheus - image: "docker.io/prom/prometheus:v2.12.0" - imagePullPolicy: IfNotPresent - args: - - '--storage.tsdb.retention=6h' - - '--config.file=/etc/prometheus/prometheus.yml' - ports: - - containerPort: 9090 - name: http - livenessProbe: - httpGet: - path: /-/healthy - port: 9090 - readinessProbe: - httpGet: - path: /-/ready - port: 9090 - resources: - requests: - cpu: 10m - - volumeMounts: - - name: config-volume - mountPath: /etc/prometheus - - mountPath: /etc/istio-certs - name: istio-certs - volumes: - - name: config-volume - configMap: - name: prometheus - - name: istio-certs - secret: - defaultMode: 420 - secretName: istio.default - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x ---- - - -apiVersion: v1 -kind: Service -metadata: - name: prometheus - namespace: istio-system - annotations: - prometheus.io/scrape: 'true' - labels: - app: prometheus - release: istio -spec: - selector: - app: prometheus - ports: - - name: http-prometheus - protocol: TCP - port: 9090 ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: prometheus - namespace: istio-system - labels: - app: prometheus - release: istio ---- - -# PrometheusOperator component is disabled. - -# Resources for Telemetry component - -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - labels: - app: mixer - release: istio - name: istio-telemetry - namespace: istio-system -spec: - maxReplicas: 5 - metrics: - - resource: - name: cpu - targetAverageUtilization: 80 - type: Resource - minReplicas: 1 - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: istio-telemetry - ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: istio-mixer-istio-system - labels: - app: istio-telemetry - release: istio -rules: -- apiGroups: ["config.istio.io"] # istio CRD watcher - resources: ["*"] - verbs: ["create", "get", "list", "watch", "patch"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["configmaps", "endpoints", "pods", "services", "namespaces", "secrets", "replicationcontrollers"] - verbs: ["get", "list", "watch"] -- apiGroups: ["extensions", "apps"] - resources: ["replicasets"] - verbs: ["get", "list", "watch"] ---- - - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: istio-mixer-admin-role-binding-istio-system - labels: - app: istio-telemetry - release: istio -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: istio-mixer-istio-system -subjects: - - kind: ServiceAccount - name: istio-mixer-service-account - namespace: istio-system ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: attributemanifest -metadata: - name: istioproxy - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - attributes: - origin.ip: - valueType: IP_ADDRESS - origin.uid: - valueType: STRING - origin.user: - valueType: STRING - request.headers: - valueType: STRING_MAP - request.id: - valueType: STRING - request.host: - valueType: STRING - request.method: - valueType: STRING - request.path: - valueType: STRING - request.url_path: - valueType: STRING - request.query_params: - valueType: STRING_MAP - request.reason: - valueType: STRING - request.referer: - valueType: STRING - request.scheme: - valueType: STRING - request.total_size: - valueType: INT64 - request.size: - valueType: INT64 - request.time: - valueType: TIMESTAMP - request.useragent: - valueType: STRING - response.code: - valueType: INT64 - response.duration: - valueType: DURATION - response.headers: - valueType: STRING_MAP - response.total_size: - valueType: INT64 - response.size: - valueType: INT64 - response.time: - valueType: TIMESTAMP - response.grpc_status: - valueType: STRING - response.grpc_message: - valueType: STRING - source.uid: - valueType: STRING - source.user: # DEPRECATED - valueType: STRING - source.principal: - valueType: STRING - destination.uid: - valueType: STRING - destination.principal: - valueType: STRING - destination.port: - valueType: INT64 - connection.event: - valueType: STRING - connection.id: - valueType: STRING - connection.received.bytes: - valueType: INT64 - connection.received.bytes_total: - valueType: INT64 - connection.sent.bytes: - valueType: INT64 - connection.sent.bytes_total: - valueType: INT64 - connection.duration: - valueType: DURATION - connection.mtls: - valueType: BOOL - connection.requested_server_name: - valueType: STRING - context.protocol: - valueType: STRING - context.proxy_error_code: - valueType: STRING - context.timestamp: - valueType: TIMESTAMP - context.time: - valueType: TIMESTAMP - # Deprecated, kept for compatibility - context.reporter.local: - valueType: BOOL - context.reporter.kind: - valueType: STRING - context.reporter.uid: - valueType: STRING - context.proxy_version: - valueType: STRING - api.service: - valueType: STRING - api.version: - valueType: STRING - api.operation: - valueType: STRING - api.protocol: - valueType: STRING - request.auth.principal: - valueType: STRING - request.auth.audiences: - valueType: STRING - request.auth.presenter: - valueType: STRING - request.auth.claims: - valueType: STRING_MAP - request.auth.raw_claims: - valueType: STRING - request.api_key: - valueType: STRING - rbac.permissive.response_code: - valueType: STRING - rbac.permissive.effective_policy_id: - valueType: STRING - check.error_code: - valueType: INT64 - check.error_message: - valueType: STRING - check.cache_hit: - valueType: BOOL - quota.cache_hit: - valueType: BOOL ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: attributemanifest -metadata: - name: kubernetes - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - attributes: - source.ip: - valueType: IP_ADDRESS - source.labels: - valueType: STRING_MAP - source.metadata: - valueType: STRING_MAP - source.name: - valueType: STRING - source.namespace: - valueType: STRING - source.owner: - valueType: STRING - source.serviceAccount: - valueType: STRING - source.services: - valueType: STRING - source.workload.uid: - valueType: STRING - source.workload.name: - valueType: STRING - source.workload.namespace: - valueType: STRING - destination.ip: - valueType: IP_ADDRESS - destination.labels: - valueType: STRING_MAP - destination.metadata: - valueType: STRING_MAP - destination.owner: - valueType: STRING - destination.name: - valueType: STRING - destination.container.name: - valueType: STRING - destination.namespace: - valueType: STRING - destination.service.uid: - valueType: STRING - destination.service.name: - valueType: STRING - destination.service.namespace: - valueType: STRING - destination.service.host: - valueType: STRING - destination.serviceAccount: - valueType: STRING - destination.workload.uid: - valueType: STRING - destination.workload.name: - valueType: STRING - destination.workload.namespace: - valueType: STRING ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: handler -metadata: - name: stdio - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledAdapter: stdio - params: - outputAsJson: false ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: accesslog - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: logentry - params: - severity: '"Info"' - timestamp: request.time - variables: - sourceIp: source.ip | ip("0.0.0.0") - sourceApp: source.labels["app"] | "" - sourcePrincipal: source.principal | "" - sourceName: source.name | "" - sourceWorkload: source.workload.name | "" - sourceNamespace: source.namespace | "" - sourceOwner: source.owner | "" - destinationApp: destination.labels["app"] | "" - destinationIp: destination.ip | ip("0.0.0.0") - destinationServiceHost: destination.service.host | request.host | "" - destinationWorkload: destination.workload.name | "" - destinationName: destination.name | "" - destinationNamespace: destination.namespace | "" - destinationOwner: destination.owner | "" - destinationPrincipal: destination.principal | "" - apiClaims: request.auth.raw_claims | "" - apiKey: request.api_key | request.headers["x-api-key"] | "" - protocol: request.scheme | context.protocol | "http" - method: request.method | "" - url: request.path | "" - responseCode: response.code | 0 - responseFlags: context.proxy_error_code | "" - responseSize: response.size | 0 - permissiveResponseCode: rbac.permissive.response_code | "none" - permissiveResponsePolicyID: rbac.permissive.effective_policy_id | "none" - requestSize: request.size | 0 - requestId: request.headers["x-request-id"] | "" - clientTraceId: request.headers["x-client-trace-id"] | "" - latency: response.duration | "0ms" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - requestedServerName: connection.requested_server_name | "" - userAgent: request.useragent | "" - responseTimestamp: response.time - receivedBytes: request.total_size | 0 - sentBytes: response.total_size | 0 - referer: request.referer | "" - httpAuthority: request.headers[":authority"] | request.host | "" - xForwardedFor: request.headers["x-forwarded-for"] | "0.0.0.0" - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - grpcStatus: response.grpc_status | "" - grpcMessage: response.grpc_message | "" - monitored_resource_type: '"global"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: tcpaccesslog - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: logentry - params: - severity: '"Info"' - timestamp: context.time | timestamp("2017-01-01T00:00:00Z") - variables: - connectionEvent: connection.event | "" - sourceIp: source.ip | ip("0.0.0.0") - sourceApp: source.labels["app"] | "" - sourcePrincipal: source.principal | "" - sourceName: source.name | "" - sourceWorkload: source.workload.name | "" - sourceNamespace: source.namespace | "" - sourceOwner: source.owner | "" - destinationApp: destination.labels["app"] | "" - destinationIp: destination.ip | ip("0.0.0.0") - destinationServiceHost: destination.service.host | "" - destinationWorkload: destination.workload.name | "" - destinationName: destination.name | "" - destinationNamespace: destination.namespace | "" - destinationOwner: destination.owner | "" - destinationPrincipal: destination.principal | "" - protocol: context.protocol | "tcp" - connectionDuration: connection.duration | "0ms" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - requestedServerName: connection.requested_server_name | "" - receivedBytes: connection.received.bytes | 0 - sentBytes: connection.sent.bytes | 0 - totalReceivedBytes: connection.received.bytes_total | 0 - totalSentBytes: connection.sent.bytes_total | 0 - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - responseFlags: context.proxy_error_code | "" - monitored_resource_type: '"global"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: stdio - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - match: context.protocol == "http" || context.protocol == "grpc" - actions: - - handler: stdio - instances: - - accesslog ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: stdiotcp - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - match: context.protocol == "tcp" - actions: - - handler: stdio - instances: - - tcpaccesslog ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: requestcount - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: "1" - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | conditional((destination.service.name | "unknown") == "unknown", "unknown", request.host) - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - request_protocol: api.protocol | context.protocol | "unknown" - response_code: response.code | 200 - response_flags: context.proxy_error_code | "-" - permissive_response_code: rbac.permissive.response_code | "none" - permissive_response_policyid: rbac.permissive.effective_policy_id | "none" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: requestduration - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: response.duration | "0ms" - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | conditional((destination.service.name | "unknown") == "unknown", "unknown", request.host) - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - request_protocol: api.protocol | context.protocol | "unknown" - response_code: response.code | 200 - response_flags: context.proxy_error_code | "-" - permissive_response_code: rbac.permissive.response_code | "none" - permissive_response_policyid: rbac.permissive.effective_policy_id | "none" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: requestsize - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: request.size | 0 - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | conditional((destination.service.name | "unknown") == "unknown", "unknown", request.host) - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - request_protocol: api.protocol | context.protocol | "unknown" - response_code: response.code | 200 - response_flags: context.proxy_error_code | "-" - permissive_response_code: rbac.permissive.response_code | "none" - permissive_response_policyid: rbac.permissive.effective_policy_id | "none" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: responsesize - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: response.size | 0 - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | conditional((destination.service.name | "unknown") == "unknown", "unknown", request.host) - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - request_protocol: api.protocol | context.protocol | "unknown" - response_code: response.code | 200 - response_flags: context.proxy_error_code | "-" - permissive_response_code: rbac.permissive.response_code | "none" - permissive_response_policyid: rbac.permissive.effective_policy_id | "none" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: tcpbytesent - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: connection.sent.bytes | 0 - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | "unknown" - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - response_flags: context.proxy_error_code | "-" - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: tcpbytereceived - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: connection.received.bytes | 0 - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | "unknown" - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - response_flags: context.proxy_error_code | "-" - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: tcpconnectionsopened - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: "1" - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | "unknown" - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - response_flags: context.proxy_error_code | "-" - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: tcpconnectionsclosed - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: metric - params: - value: "1" - dimensions: - reporter: conditional((context.reporter.kind | "inbound") == "outbound", "source", "destination") - source_workload: source.workload.name | "unknown" - source_workload_namespace: source.workload.namespace | "unknown" - source_principal: source.principal | "unknown" - source_app: source.labels["app"] | "unknown" - source_version: source.labels["version"] | "unknown" - destination_workload: destination.workload.name | "unknown" - destination_workload_namespace: destination.workload.namespace | "unknown" - destination_principal: destination.principal | "unknown" - destination_app: destination.labels["app"] | "unknown" - destination_version: destination.labels["version"] | "unknown" - destination_service: destination.service.host | "unknown" - destination_service_name: destination.service.name | "unknown" - destination_service_namespace: destination.service.namespace | "unknown" - connection_security_policy: conditional((context.reporter.kind | "inbound") == "outbound", "unknown", conditional(connection.mtls | false, "mutual_tls", "none")) - response_flags: context.proxy_error_code | "-" - monitored_resource_type: '"UNSPECIFIED"' ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: handler -metadata: - name: prometheus - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledAdapter: prometheus - params: - metricsExpirationPolicy: - metricsExpiryDuration: "10m" - metrics: - - name: requests_total - instance_name: requestcount.instance.istio-system - kind: COUNTER - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - request_protocol - - response_code - - response_flags - - permissive_response_code - - permissive_response_policyid - - connection_security_policy - - name: request_duration_seconds - instance_name: requestduration.instance.istio-system - kind: DISTRIBUTION - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - request_protocol - - response_code - - response_flags - - permissive_response_code - - permissive_response_policyid - - connection_security_policy - buckets: - explicit_buckets: - bounds: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10] - - name: request_bytes - instance_name: requestsize.instance.istio-system - kind: DISTRIBUTION - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - request_protocol - - response_code - - response_flags - - permissive_response_code - - permissive_response_policyid - - connection_security_policy - buckets: - exponentialBuckets: - numFiniteBuckets: 8 - scale: 1 - growthFactor: 10 - - name: response_bytes - instance_name: responsesize.instance.istio-system - kind: DISTRIBUTION - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - request_protocol - - response_code - - response_flags - - permissive_response_code - - permissive_response_policyid - - connection_security_policy - buckets: - exponentialBuckets: - numFiniteBuckets: 8 - scale: 1 - growthFactor: 10 - - name: tcp_sent_bytes_total - instance_name: tcpbytesent.instance.istio-system - kind: COUNTER - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - connection_security_policy - - response_flags - - name: tcp_received_bytes_total - instance_name: tcpbytereceived.instance.istio-system - kind: COUNTER - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - connection_security_policy - - response_flags - - name: tcp_connections_opened_total - instance_name: tcpconnectionsopened.instance.istio-system - kind: COUNTER - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - connection_security_policy - - response_flags - - name: tcp_connections_closed_total - instance_name: tcpconnectionsclosed.instance.istio-system - kind: COUNTER - label_names: - - reporter - - source_app - - source_principal - - source_workload - - source_workload_namespace - - source_version - - destination_app - - destination_principal - - destination_workload - - destination_workload_namespace - - destination_version - - destination_service - - destination_service_name - - destination_service_namespace - - connection_security_policy - - response_flags ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: promhttp - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - match: (context.protocol == "http" || context.protocol == "grpc") && (match((request.useragent | "-"), "kube-probe*") == false) && (match((request.useragent | "-"), "Prometheus*") == false) - actions: - - handler: prometheus - instances: - - requestcount - - requestduration - - requestsize - - responsesize ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: promtcp - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - match: context.protocol == "tcp" - actions: - - handler: prometheus - instances: - - tcpbytesent - - tcpbytereceived ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: promtcpconnectionopen - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - match: context.protocol == "tcp" && ((connection.event | "na") == "open") - actions: - - handler: prometheus - instances: - - tcpconnectionsopened ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: promtcpconnectionclosed - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - match: context.protocol == "tcp" && ((connection.event | "na") == "close") - actions: - - handler: prometheus - instances: - - tcpconnectionsclosed ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: handler -metadata: - name: kubernetesenv - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledAdapter: kubernetesenv - params: - # when running from mixer root, use the following config after adding a - # symbolic link to a kubernetes config file via: - # - # $ ln -s ~/.kube/config mixer/adapter/kubernetes/kubeconfig - # - # kubeconfig_path: "mixer/adapter/kubernetes/kubeconfig" ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: kubeattrgenrulerule - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - actions: - - handler: kubernetesenv - instances: - - attributes ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: rule -metadata: - name: tcpkubeattrgenrulerule - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - match: context.protocol == "tcp" - actions: - - handler: kubernetesenv - instances: - - attributes ---- - - -apiVersion: "config.istio.io/v1alpha2" -kind: instance -metadata: - name: attributes - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - compiledTemplate: kubernetes - params: - # Pass the required attribute data to the adapter - source_uid: source.uid | "" - source_ip: source.ip | ip("0.0.0.0") # default to unspecified ip addr - destination_uid: destination.uid | "" - destination_port: destination.port | 0 - attributeBindings: - # Fill the new attributes from the adapter produced output. - # $out refers to an instance of OutputTemplate message - source.ip: $out.source_pod_ip | ip("0.0.0.0") - source.uid: $out.source_pod_uid | "unknown" - source.labels: $out.source_labels | emptyStringMap() - source.name: $out.source_pod_name | "unknown" - source.namespace: $out.source_namespace | "default" - source.owner: $out.source_owner | "unknown" - source.serviceAccount: $out.source_service_account_name | "unknown" - source.workload.uid: $out.source_workload_uid | "unknown" - source.workload.name: $out.source_workload_name | "unknown" - source.workload.namespace: $out.source_workload_namespace | "unknown" - destination.ip: $out.destination_pod_ip | ip("0.0.0.0") - destination.uid: $out.destination_pod_uid | "unknown" - destination.labels: $out.destination_labels | emptyStringMap() - destination.name: $out.destination_pod_name | "unknown" - destination.container.name: $out.destination_container_name | "unknown" - destination.namespace: $out.destination_namespace | "default" - destination.owner: $out.destination_owner | "unknown" - destination.serviceAccount: $out.destination_service_account_name | "unknown" - destination.workload.uid: $out.destination_workload_uid | "unknown" - destination.workload.name: $out.destination_workload_name | "unknown" - destination.workload.namespace: $out.destination_workload_namespace | "unknown" ---- - - -apiVersion: networking.istio.io/v1alpha3 -kind: DestinationRule -metadata: - name: istio-telemetry - namespace: istio-system - labels: - app: istio-telemetry - release: istio -spec: - host: istio-telemetry.istio-system.svc.cluster.local - trafficPolicy: - portLevelSettings: - - port: - number: 15004 # grpc-mixer-mtls - tls: - mode: ISTIO_MUTUAL - - port: - number: 9091 # grpc-mixer - tls: - mode: DISABLE - connectionPool: - http: - http2MaxRequests: 10000 - maxRequestsPerConnection: 10000 ---- - - -apiVersion: v1 -kind: ConfigMap -metadata: - namespace: istio-system - name: telemetry-envoy-config - labels: - release: istio -data: - # Explicitly defined - moved from istio/istio/pilot/docker. - envoy.yaml.tmpl: |- - admin: - access_log_path: /dev/null - address: - socket_address: - address: 127.0.0.1 - port_value: 15000 - stats_config: - use_all_default_tags: false - stats_tags: - - tag_name: cluster_name - regex: '^cluster\.((.+?(\..+?\.svc\.cluster\.local)?)\.)' - - tag_name: tcp_prefix - regex: '^tcp\.((.*?)\.)\w+?$' - - tag_name: response_code - regex: '_rq(_(\d{3}))$' - - tag_name: response_code_class - regex: '_rq(_(\dxx))$' - - tag_name: http_conn_manager_listener_prefix - regex: '^listener(?=\.).*?\.http\.(((?:[_.[:digit:]]*|[_\[\]aAbBcCdDeEfF[:digit:]]*))\.)' - - tag_name: http_conn_manager_prefix - regex: '^http\.(((?:[_.[:digit:]]*|[_\[\]aAbBcCdDeEfF[:digit:]]*))\.)' - - tag_name: listener_address - regex: '^listener\.(((?:[_.[:digit:]]*|[_\[\]aAbBcCdDeEfF[:digit:]]*))\.)' - - static_resources: - clusters: - - name: prometheus_stats - type: STATIC - connect_timeout: 0.250s - lb_policy: ROUND_ROBIN - hosts: - - socket_address: - protocol: TCP - address: 127.0.0.1 - port_value: 15000 - - - name: inbound_9092 - circuit_breakers: - thresholds: - - max_connections: 100000 - max_pending_requests: 100000 - max_requests: 100000 - max_retries: 3 - connect_timeout: 1.000s - hosts: - - pipe: - path: /sock/mixer.socket - http2_protocol_options: {} - - - name: out.galley.15019 - http2_protocol_options: {} - connect_timeout: 1.000s - type: STRICT_DNS - - circuit_breakers: - thresholds: - - max_connections: 100000 - max_pending_requests: 100000 - max_requests: 100000 - max_retries: 3 - - tls_context: - common_tls_context: - tls_certificates: - - certificate_chain: - filename: /etc/certs/cert-chain.pem - private_key: - filename: /etc/certs/key.pem - validation_context: - trusted_ca: - filename: /etc/certs/root-cert.pem - verify_subject_alt_name: - - spiffe://cluster.local/ns/istio-system/sa/istio-galley-service-account - - hosts: - - socket_address: - address: istio-galley.istio-system - port_value: 15019 - - - listeners: - - name: "15090" - address: - socket_address: - protocol: TCP - address: 0.0.0.0 - port_value: 15090 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - codec_type: AUTO - stat_prefix: stats - route_config: - virtual_hosts: - - name: backend - domains: - - '*' - routes: - - match: - prefix: /stats/prometheus - route: - cluster: prometheus_stats - http_filters: - - name: envoy.router - - - name: "15004" - address: - socket_address: - address: 0.0.0.0 - port_value: 15004 - filter_chains: - - filters: - - config: - codec_type: HTTP2 - http2_protocol_options: - max_concurrent_streams: 1073741824 - generate_request_id: true - http_filters: - - config: - default_destination_service: istio-telemetry.istio-system.svc.cluster.local - service_configs: - istio-telemetry.istio-system.svc.cluster.local: - disable_check_calls: true - {{- if .DisableReportCalls }} - disable_report_calls: true - {{- end }} - mixer_attributes: - attributes: - destination.service.host: - string_value: istio-telemetry.istio-system.svc.cluster.local - destination.service.uid: - string_value: istio://istio-system/services/istio-telemetry - destination.service.name: - string_value: istio-telemetry - destination.service.namespace: - string_value: istio-system - destination.uid: - string_value: kubernetes://{{ .PodName }}.istio-system - destination.namespace: - string_value: istio-system - destination.ip: - bytes_value: {{ .PodIP }} - destination.port: - int64_value: 15004 - context.reporter.kind: - string_value: inbound - context.reporter.uid: - string_value: kubernetes://{{ .PodName }}.istio-system - transport: - check_cluster: mixer_check_server - report_cluster: inbound_9092 - name: mixer - - name: envoy.router - route_config: - name: "15004" - virtual_hosts: - - domains: - - '*' - name: istio-telemetry.istio-system.svc.cluster.local - routes: - - decorator: - operation: Report - match: - prefix: / - route: - cluster: inbound_9092 - timeout: 0.000s - stat_prefix: "15004" - name: envoy.http_connection_manager - - - name: "9091" - address: - socket_address: - address: 0.0.0.0 - port_value: 9091 - filter_chains: - - filters: - - config: - codec_type: HTTP2 - http2_protocol_options: - max_concurrent_streams: 1073741824 - generate_request_id: true - http_filters: - - config: - default_destination_service: istio-telemetry.istio-system.svc.cluster.local - service_configs: - istio-telemetry.istio-system.svc.cluster.local: - disable_check_calls: true - {{- if .DisableReportCalls }} - disable_report_calls: true - {{- end }} - mixer_attributes: - attributes: - destination.service.host: - string_value: istio-telemetry.istio-system.svc.cluster.local - destination.service.uid: - string_value: istio://istio-system/services/istio-telemetry - destination.service.name: - string_value: istio-telemetry - destination.service.namespace: - string_value: istio-system - destination.uid: - string_value: kubernetes://{{ .PodName }}.istio-system - destination.namespace: - string_value: istio-system - destination.ip: - bytes_value: {{ .PodIP }} - destination.port: - int64_value: 9091 - context.reporter.kind: - string_value: inbound - context.reporter.uid: - string_value: kubernetes://{{ .PodName }}.istio-system - transport: - check_cluster: mixer_check_server - report_cluster: inbound_9092 - name: mixer - - name: envoy.router - route_config: - name: "9091" - virtual_hosts: - - domains: - - '*' - name: istio-telemetry.istio-system.svc.cluster.local - routes: - - decorator: - operation: Report - match: - prefix: / - route: - cluster: inbound_9092 - timeout: 0.000s - stat_prefix: "9091" - name: envoy.http_connection_manager - - - name: "local.15019" - address: - socket_address: - address: 127.0.0.1 - port_value: 15019 - filter_chains: - - filters: - - name: envoy.http_connection_manager - config: - codec_type: HTTP2 - stat_prefix: "15019" - http2_protocol_options: - max_concurrent_streams: 1073741824 - - access_log: - - name: envoy.file_access_log - config: - path: /dev/stdout - - http_filters: - - name: envoy.router - - route_config: - name: "15019" - - virtual_hosts: - - name: istio-galley - - domains: - - '*' - - routes: - - match: - prefix: / - route: - cluster: out.galley.15019 - timeout: 0.000s ---- - - -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: istio-mixer - istio: mixer - release: istio - name: istio-telemetry - namespace: istio-system -spec: - replicas: 1 - selector: - matchLabels: - istio: mixer - istio-mixer-type: telemetry - strategy: - rollingUpdate: - maxSurge: 100% - maxUnavailable: 25% - template: - metadata: - annotations: - sidecar.istio.io/inject: "false" - labels: - app: telemetry - istio: mixer - istio-mixer-type: telemetry - spec: - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - weight: 2 - - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - weight: 2 - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - containers: - - args: - - --monitoringPort=15014 - - --address - - tcp://0.0.0.0:9091 - - --log_output_level=default:info - - --configStoreURL=mcp://istio-galley.istio-system.svc:9901 - - --configDefaultNamespace=istio-system - - --useAdapterCRDs=false - - --useTemplateCRDs=false - - --trace_zipkin_url=http://zipkin.istio-system:9411/api/v1/spans - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: GOMAXPROCS - value: "6" - image: docker.io/istio/mixer:1.4.2 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /version - port: 15014 - initialDelaySeconds: 5 - periodSeconds: 5 - name: mixer - ports: - - containerPort: 9091 - - containerPort: 15014 - - containerPort: 42422 - resources: - limits: - cpu: 4800m - memory: 4G - requests: - cpu: 50m - memory: 100Mi - volumeMounts: - - mountPath: /etc/certs - name: istio-certs - readOnly: true - - mountPath: /sock - name: uds-socket - - mountPath: /var/run/secrets/istio.io/telemetry/adapter - name: telemetry-adapter-secret - readOnly: true - serviceAccountName: istio-mixer-service-account - volumes: - - name: istio-certs - secret: - optional: true - secretName: istio.istio-mixer-service-account - - emptyDir: {} - name: uds-socket - - name: telemetry-adapter-secret - secret: - optional: true - secretName: telemetry-adapter-secret - - configMap: - name: telemetry-envoy-config - name: telemetry-envoy-config - ---- - - -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: istio-telemetry - namespace: istio-system - labels: - app: telemetry - release: istio - istio: mixer - istio-mixer-type: telemetry -spec: - minAvailable: 1 - selector: - matchLabels: - app: telemetry - release: istio - istio: mixer - istio-mixer-type: telemetry ---- - - -apiVersion: v1 -kind: Service -metadata: - name: istio-telemetry - namespace: istio-system - labels: - app: mixer - istio: mixer - release: istio -spec: - ports: - - name: grpc-mixer - port: 9091 - - name: grpc-mixer-mtls - port: 15004 - - name: http-monitoring - port: 15014 - - name: prometheus - port: 42422 - selector: - istio: mixer - istio-mixer-type: telemetry ---- - - -apiVersion: v1 -kind: ServiceAccount -metadata: - name: istio-mixer-service-account - namespace: istio-system - labels: - app: istio-telemetry - release: istio ---- - -# Resources for Tracing component - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: istio-tracing - namespace: istio-system - labels: - app: jaeger - release: istio -spec: - selector: - matchLabels: - app: jaeger - template: - metadata: - labels: - app: jaeger - release: istio - annotations: - sidecar.istio.io/inject: "false" - prometheus.io/scrape: "true" - prometheus.io/port: "14269" - spec: - containers: - - name: jaeger - image: "docker.io/jaegertracing/all-in-one:1.14" - imagePullPolicy: IfNotPresent - ports: - - containerPort: 9411 - - containerPort: 16686 - - containerPort: 14250 - - containerPort: 14267 - - containerPort: 14268 - - containerPort: 14269 - - containerPort: 5775 - protocol: UDP - - containerPort: 6831 - protocol: UDP - - containerPort: 6832 - protocol: UDP - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: BADGER_EPHEMERAL - value: "false" - - name: SPAN_STORAGE_TYPE - value: "badger" - - name: BADGER_DIRECTORY_VALUE - value: "/badger/data" - - name: BADGER_DIRECTORY_KEY - value: "/badger/key" - - name: COLLECTOR_ZIPKIN_HTTP_PORT - value: "9411" - - name: MEMORY_MAX_TRACES - value: "50000" - - name: QUERY_BASE_PATH - value: /jaeger - livenessProbe: - httpGet: - path: / - port: 14269 - readinessProbe: - httpGet: - path: / - port: 14269 - volumeMounts: - - name: data - mountPath: /badger - resources: - requests: - cpu: 10m - - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - ppc64le - - s390x - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - amd64 - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - ppc64le - - weight: 2 - preference: - matchExpressions: - - key: beta.kubernetes.io/arch - operator: In - values: - - s390x - volumes: - - name: data - emptyDir: {} ---- - - -apiVersion: v1 -kind: Service -metadata: - name: jaeger-query - namespace: istio-system - annotations: - labels: - app: jaeger - jaeger-infra: jaeger-service - release: istio -spec: - ports: - - name: query-http - port: 16686 - protocol: TCP - targetPort: 16686 - selector: - app: jaeger ---- - - -apiVersion: v1 -kind: Service -metadata: - name: jaeger-collector - namespace: istio-system - labels: - app: jaeger - jaeger-infra: collector-service - release: istio -spec: - ports: - - name: jaeger-collector-tchannel - port: 14267 - protocol: TCP - targetPort: 14267 - - name: jaeger-collector-http - port: 14268 - targetPort: 14268 - protocol: TCP - - name: jaeger-collector-grpc - port: 14250 - targetPort: 14250 - protocol: TCP - selector: - app: jaeger - type: ClusterIP ---- - - -apiVersion: v1 -kind: Service -metadata: - name: jaeger-agent - namespace: istio-system - labels: - app: jaeger - jaeger-infra: agent-service - release: istio -spec: - ports: - - name: agent-zipkin-thrift - port: 5775 - protocol: UDP - targetPort: 5775 - - name: agent-compact - port: 6831 - protocol: UDP - targetPort: 6831 - - name: agent-binary - port: 6832 - protocol: UDP - targetPort: 6832 - clusterIP: None - selector: - app: jaeger ---- - - -apiVersion: v1 -kind: Service -metadata: - name: zipkin - namespace: istio-system - labels: - app: jaeger - release: istio -spec: - ports: - - port: 9411 - targetPort: 9411 - protocol: TCP - name: http-query - selector: - app: jaeger ---- - - -apiVersion: v1 -kind: Service -metadata: - name: tracing - namespace: istio-system - annotations: - labels: - app: jaeger - release: istio -spec: - type: ClusterIP - ports: - - name: http-query - port: 80 - protocol: TCP - - targetPort: 16686 - - selector: - app: jaeger ---- - diff --git a/testing/scripts/kind_test_all.sh b/testing/scripts/kind_test_all.sh index 516a851721..9ecbf94a75 100755 --- a/testing/scripts/kind_test_all.sh +++ b/testing/scripts/kind_test_all.sh @@ -154,7 +154,7 @@ if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then fi echo "Files changed in misc folders:" - git --no-pager diff --exit-code --name-only origin/master ../../components/seldon-request-logger ../../components/storage-initializer + git --no-pager diff --exit-code --name-only origin/master ../../components/seldon-request-logger ../../components/storage-initializer ../../components/routers/epsilon-greedy MISC_MODIFIED=$? if [[ $MISC_MODIFIED -gt 0 ]]; then make kind_build_misc diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index f0d1efa528..00864cbd5e 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -492,12 +492,7 @@ def rest_request_ambassador_auth( def grpc_request_ambassador( - deployment_name, - namespace, - endpoint="localhost:8004", - data_size=5, - rows=1, - data=None, + deployment_name, namespace, endpoint=API_AMBASSADOR, data_size=5, rows=1, data=None ): if data is None: shape, arr = create_random_data(data_size, rows) @@ -524,7 +519,7 @@ def grpc_request_ambassador( def grpc_request_ambassador_metadata( - deployment_name, namespace, endpoint="localhost:8004", model_name=None + deployment_name, namespace, endpoint=API_AMBASSADOR, model_name=None ): if model_name is None: request = empty_pb2.Empty() diff --git a/testing/scripts/test_notebooks.py b/testing/scripts/test_notebooks.py index bb67fb3fc7..d7691044fb 100644 --- a/testing/scripts/test_notebooks.py +++ b/testing/scripts/test_notebooks.py @@ -75,13 +75,6 @@ def test_ambassador_shadow(self): def test_ambassador_custom(self): create_and_run_script("../../examples/ambassador/custom", "ambassador_custom") - # - # Istio Examples - # - - def test_istio_canary(self): - create_and_run_script("../../examples/istio/canary_update", "canary") - # # KEDA Examples # diff --git a/testing/scripts/test_prepackaged_servers.py b/testing/scripts/test_prepackaged_servers.py index 7caa22281c..81229d2144 100644 --- a/testing/scripts/test_prepackaged_servers.py +++ b/testing/scripts/test_prepackaged_servers.py @@ -7,6 +7,7 @@ wait_for_rollout, initial_rest_request, rest_request_ambassador, + grpc_request_ambassador, retry_run, create_random_data, wait_for_status, @@ -16,6 +17,9 @@ from e2e_utils import v2_protocol from e2e_utils.models import deploy_model from conftest import SELDON_E2E_TESTS_USE_EXECUTOR +import numpy as np +import json +from google.protobuf import json_format skipif_engine = pytest.mark.skipif( not SELDON_E2E_TESTS_USE_EXECUTOR, reason="Not supported by the Java engine" @@ -46,6 +50,12 @@ def test_sklearn(self, namespace): assert res["name"] == "iris" assert res["versions"] == ["iris/v1"] + r = grpc_request_ambassador( + "sklearn", namespace, data=np.array([[0.1, 0.2, 0.3, 0.4]]) + ) + res = json.loads(json_format.MessageToJson(r)) + logging.info(res) + logging.warning("Success for test_prepack_sklearn") run(f"kubectl delete -f {spec} -n {namespace}", shell=True) @@ -74,7 +84,7 @@ def test_sklearn_v2(self, namespace): "datatype": "FP32", "data": [[0.1, 0.2, 0.3, 0.4]], } - ], + ] }, ) assert r.status_code == 200 @@ -119,6 +129,12 @@ def test_xgboost(self, namespace): assert res["name"] == "xgboost-iris" assert res["versions"] == ["xgboost-iris/v1"] + r = grpc_request_ambassador( + "xgboost", namespace, data=np.array([[0.1, 0.2, 0.3, 0.4]]) + ) + res = json.loads(json_format.MessageToJson(r)) + logging.info(res) + logging.warning("Success for test_prepack_xgboost") run(f"kubectl delete -f {spec} -n {namespace}", shell=True) @@ -147,7 +163,7 @@ def test_xgboost_v2(self, namespace): "datatype": "FP32", "data": [[0.1, 0.2, 0.3, 0.4]], } - ], + ] }, ) assert r.status_code == 200 diff --git a/testing/scripts/test_s2i_java.py b/testing/scripts/test_s2i_java.py index c72cb75c3c..15a3dc205e 100644 --- a/testing/scripts/test_s2i_java.py +++ b/testing/scripts/test_s2i_java.py @@ -12,16 +12,16 @@ S2I_JNI_PARAMETERS = { "s2i_folder": JAVA_S2I_FOLDER, - "s2i_image": "seldonio/s2i-java-jni-build:0.3.0", + "s2i_image": "seldonio/s2i-java-jni-build:0.4.0", "image_name": "seldonio/test-s2i-java-jni:0.2.0", - "s2i_runtime_image": "seldonio/s2i-java-jni-runtime:0.3.0", + "s2i_runtime_image": "seldonio/s2i-java-jni-runtime:0.4.0", } S2I_JAVA_PARAMETERS = { "s2i_folder": JAVA_S2I_FOLDER, - "s2i_image": "seldonio/seldon-core-s2i-java-build:0.3.0", + "s2i_image": "seldonio/seldon-core-s2i-java-build:0.4.0", "image_name": "seldonio/test-s2i-java:0.2.0", - "s2i_runtime_image": "seldonio/seldon-core-s2i-java-runtime:0.3.0", + "s2i_runtime_image": "seldonio/seldon-core-s2i-java-runtime:0.4.0", } diff --git a/testing/scripts/test_s2i_python.py b/testing/scripts/test_s2i_python.py index cf106c032d..25d7fd47ef 100644 --- a/testing/scripts/test_s2i_python.py +++ b/testing/scripts/test_s2i_python.py @@ -12,111 +12,75 @@ ) import logging -S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment_#API# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE#_#API#:0.1" -IMAGE_NAME = "seldonio/test#TYPE#_#API#:0.1" +S2I_CREATE = "cd ../s2i/python/#TYPE# && s2i build -E environment#SUFFIX# . seldonio/seldon-core-s2i-python3:#VERSION# seldonio/test#TYPE##SUFFIX#:0.1" +IMAGE_NAME = "seldonio/test#TYPE##SUFFIX#:0.1" -def create_s2I_image(s2i_python_version, component_type, api_type): +def create_s2I_image(s2i_python_version, component_type, suffix): cmd = ( S2I_CREATE.replace("#TYPE#", component_type) - .replace("#API#", api_type) + .replace("#SUFFIX#", suffix) .replace("#VERSION#", s2i_python_version) ) logging.warning(cmd) run(cmd, shell=True, check=True) -def kind_push_s2i_image(component_type, api_type): - img = get_image_name(component_type, api_type) +def kind_push_s2i_image(component_type, suffix): + img = get_image_name(component_type, suffix) cmd = "kind load docker-image " + img logging.warning(cmd) run(cmd, shell=True, check=True) -def get_image_name(component_type, api_type): - return IMAGE_NAME.replace("#TYPE#", component_type).replace("#API#", api_type) +def get_image_name(component_type, suffix): + return IMAGE_NAME.replace("#TYPE#", component_type).replace("#SUFFIX#", suffix) -def create_push_s2i_image(s2i_python_version, component_type, api_type): - create_s2I_image(s2i_python_version, component_type, api_type) - kind_push_s2i_image(component_type, api_type) +def create_push_s2i_image(s2i_python_version, component_type, suffix): + create_s2I_image(s2i_python_version, component_type, suffix) + kind_push_s2i_image(component_type, suffix) @pytest.mark.sequential @pytest.mark.usefixtures("s2i_python_version") class TestPythonS2i(object): - def test_build_router_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "router", "rest") - img = get_image_name("router", "rest") - run("docker run -d --rm --name 'router-rest' " + img, shell=True, check=True) + def test_build_router(self, s2i_python_version): + create_s2I_image(s2i_python_version, "router", "") + img = get_image_name("router", "") + run("docker run -d --rm --name 'router' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f router-rest", shell=True, check=True) + run("docker rm -f router", shell=True, check=True) - def test_build_router_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "router", "grpc") - img = get_image_name("router", "grpc") - run("docker run -d --rm --name 'router-grpc' " + img, shell=True, check=True) + def test_build_model(self, s2i_python_version): + create_s2I_image(s2i_python_version, "model", "") + img = get_image_name("model", "") + run("docker run -d --rm --name 'model' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f router-grpc", shell=True, check=True) + run("docker rm -f model", shell=True, check=True) - def test_build_model_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "model", "rest") - img = get_image_name("model", "rest") - run("docker run -d --rm --name 'model-rest' " + img, shell=True, check=True) + def test_build_transformer(self, s2i_python_version): + create_s2I_image(s2i_python_version, "transformer", "") + img = get_image_name("transformer", "") + run("docker run -d --rm --name 'transformer' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f model-rest", shell=True, check=True) + run("docker rm -f transformer", shell=True, check=True) - def test_build_model_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "model", "grpc") - img = get_image_name("model", "grpc") - run("docker run -d --rm --name 'model-grpc' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f model-grpc", shell=True, check=True) - - def test_build_transformer_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "transformer", "rest") - img = get_image_name("transformer", "rest") - run( - "docker run -d --rm --name 'transformer-rest' " + img, - shell=True, - check=True, - ) - time.sleep(2) - run("docker rm -f transformer-rest", shell=True, check=True) - - def test_build_transformer_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "transformer", "grpc") - img = get_image_name("transformer", "grpc") - run( - "docker run -d --rm --name 'transformer-grpc' " + img, - shell=True, - check=True, - ) - time.sleep(2) - run("docker rm -f transformer-grpc", shell=True, check=True) - - def test_build_combiner_rest(self, s2i_python_version): - create_s2I_image(s2i_python_version, "combiner", "rest") - img = get_image_name("combiner", "rest") + def test_build_combiner(self, s2i_python_version): + create_s2I_image(s2i_python_version, "combiner", "") + img = get_image_name("combiner", "") logging.warning(img) - run("docker run -d --rm --name 'combiner-rest' " + img, shell=True, check=True) - time.sleep(2) - run("docker rm -f combiner-rest", shell=True, check=True) - - def test_build_combiner_grpc(self, s2i_python_version): - create_s2I_image(s2i_python_version, "combiner", "grpc") - img = get_image_name("combiner", "grpc") - run("docker run -d --rm --name 'combiner-grpc' " + img, shell=True, check=True) + run("docker run -d --rm --name 'combiner' " + img, shell=True, check=True) time.sleep(2) - run("docker rm -f combiner-grpc", shell=True, check=True) + run("docker rm -f combiner", shell=True, check=True) @pytest.mark.sequential @pytest.mark.usefixtures("namespace") @pytest.mark.usefixtures("s2i_python_version") class TestPythonS2iK8s(object): - def test_model_rest(self, namespace, s2i_python_version): - create_push_s2i_image(s2i_python_version, "model", "rest") + def test_model(self, namespace, s2i_python_version): + create_push_s2i_image(s2i_python_version, "model", "") retry_run(f"kubectl apply -f ../resources/s2i_python_model.json -n {namespace}") wait_for_status("mymodel", namespace) wait_for_rollout("mymodel", namespace) @@ -134,7 +98,7 @@ def test_model_rest(self, namespace, s2i_python_version): ) def test_model_rest_non200(self, namespace, s2i_python_version): - create_push_s2i_image(s2i_python_version, "model", "rest_non200") + create_push_s2i_image(s2i_python_version, "model", "_non200") retry_run( f"kubectl apply -f ../resources/s2i_python_model_non200.json -n {namespace}" ) @@ -155,8 +119,8 @@ def test_model_rest_non200(self, namespace, s2i_python_version): shell=True, ) - def test_input_transformer_rest(self, namespace, s2i_python_version): - create_push_s2i_image(s2i_python_version, "transformer", "rest") + def test_input_transformer(self, namespace, s2i_python_version): + create_push_s2i_image(s2i_python_version, "transformer", "") retry_run( f"kubectl apply -f ../resources/s2i_python_transformer.json -n {namespace}" ) @@ -175,8 +139,8 @@ def test_input_transformer_rest(self, namespace, s2i_python_version): shell=True, ) - def test_output_transformer_rest(self, namespace, s2i_python_version): - create_push_s2i_image(s2i_python_version, "transformer", "rest") + def test_output_transformer(self, namespace, s2i_python_version): + create_push_s2i_image(s2i_python_version, "transformer", "") retry_run( f"kubectl apply -f ../resources/s2i_python_output_transformer.json -n {namespace}" ) @@ -195,9 +159,9 @@ def test_output_transformer_rest(self, namespace, s2i_python_version): shell=True, ) - def test_router_rest(self, namespace, s2i_python_version): - create_push_s2i_image(s2i_python_version, "model", "rest") - create_push_s2i_image(s2i_python_version, "router", "rest") + def test_router(self, namespace, s2i_python_version): + create_push_s2i_image(s2i_python_version, "model", "") + create_push_s2i_image(s2i_python_version, "router", "") retry_run( f"kubectl apply -f ../resources/s2i_python_router.json -n {namespace}" ) @@ -216,9 +180,9 @@ def test_router_rest(self, namespace, s2i_python_version): shell=True, ) - def test_combiner_rest(self, namespace, s2i_python_version): - create_push_s2i_image(s2i_python_version, "model", "rest") - create_push_s2i_image(s2i_python_version, "combiner", "rest") + def test_combiner(self, namespace, s2i_python_version): + create_push_s2i_image(s2i_python_version, "model", "") + create_push_s2i_image(s2i_python_version, "combiner", "") retry_run( f"kubectl apply -f ../resources/s2i_python_combiner.json -n {namespace}" ) diff --git a/testing/scripts/test_tracing.py b/testing/scripts/test_tracing.py index bda82680b8..2a28536014 100644 --- a/testing/scripts/test_tracing.py +++ b/testing/scripts/test_tracing.py @@ -54,6 +54,8 @@ def test_tracing_rest(namespace): pod_names = get_pod_names(deployment_name, namespace) pod_name = pod_names[0] + print("deployment name", deployment_name, "pod name", pod_name) + # The engine and the executor identify as different services and different # operations against Jaeger. We need to consider both. service = "executor" diff --git a/wrappers/s2i/python/s2i/bin/assemble b/wrappers/s2i/python/s2i/bin/assemble index fd116faedb..40c3dcf074 100755 --- a/wrappers/s2i/python/s2i/bin/assemble +++ b/wrappers/s2i/python/s2i/bin/assemble @@ -19,12 +19,6 @@ if [[ -z "$MODEL_NAME" ]]; then exit 1 fi -if [[ -z "$API_TYPE" ]]; then - - echo "Failed to find required env var API_TYPE, should be either REST or GRPC." - exit 1 -fi - if [[ -z "$SERVICE_TYPE" ]]; then echo "Failed to find required env var SERVICE_TYPE, should be one of MODEL, ROUTER, TRANSFORMER, COMBINER." diff --git a/wrappers/s2i/python/s2i/bin/run b/wrappers/s2i/python/s2i/bin/run index e65369cbd6..c7b3400509 100755 --- a/wrappers/s2i/python/s2i/bin/run +++ b/wrappers/s2i/python/s2i/bin/run @@ -22,9 +22,9 @@ function _is_env_present() { } #check environment vars -if [[ -z "$MODEL_NAME" || -z "$API_TYPE" || -z "$SERVICE_TYPE" || -z "$PERSISTENCE" ]]; then +if [[ -z "$MODEL_NAME" || -z "$SERVICE_TYPE" || -z "$PERSISTENCE" ]]; then - echo "Failed to find required env vars MODEL_NAME, API_TYPE, SERVICE_TYPE, PERSISTENCE" + echo "Failed to find required env vars MODEL_NAME, SERVICE_TYPE, PERSISTENCE" exit 1 else @@ -44,6 +44,6 @@ else fi echo "starting microservice" - exec seldon-core-microservice $MODEL_NAME $API_TYPE --service-type $SERVICE_TYPE --persistence $PERSISTENCE + exec seldon-core-microservice $MODEL_NAME --service-type $SERVICE_TYPE --persistence $PERSISTENCE fi