feat(pdb): Add support for managing PDBs

Add support for managing PDBs through the SeldonSpec. Contributes to #2508 Signed-off-by: Nick Groszewski <[email protected]>
SeldonIO · Oct 2, 2020 · df82a53 · df82a53
1 parent 490651c
commit df82a53
Show file tree

Hide file tree

Showing 12 changed files with 1,450 additions and 3,747 deletions.
diff --git a/README.md b/README.md
@@ -328,6 +328,7 @@ Below are some of the core components together with link to the logs that provid
 * [Payload Logging with ELK ](https://docs.seldon.io/projects/seldon-core/en/latest/analytics/logging.html)
 * [Distributed Tracing with Jaeger ](https://docs.seldon.io/projects/seldon-core/en/latest/graph/distributed-tracing.html)
 * [Replica Scaling ](https://docs.seldon.io/projects/seldon-core/en/latest/graph/scaling.html)
+* [Budgeting Disruptions ](https://docs.seldon.io/projects/seldon-core/en/latest/graph/disruption-budgets.html
 * [Custom Inference Servers](https://docs.seldon.io/projects/seldon-core/en/latest/servers/custom.html)
 
 ### Advanced Inference

diff --git a/doc/source/graph/disruption-budgets.md b/doc/source/graph/disruption-budgets.md
@@ -0,0 +1,39 @@
+# Budgeting Disruptions
+
+High availability is an important aspect in running production systems.
+To this end, you can add Pod Disruption Budget Specifications to the Pod Template Specifications you create.
+Depending on how you want your application to handle disruptions, you can define your disruption budget accordingly.
+
+An example Seldon Deployment with disruption budgets defined can be seen below:
+
+```yaml
+apiVersion: machinelearning.seldon.io/v1
+kind: SeldonDeployment
+metadata:
+  name: seldon-model
+spec:
+  name: test-deployment
+  replicas: 2
+  predictors:
+  - componentSpecs:
+    - pdbSpec:
+        minAvailable: 90%
+      spec:
+        containers:
+        - image: seldonio/mock_classifier_rest:1.3
+          imagePullPolicy: IfNotPresent
+          name: classifier
+          resources:
+            requests:
+              cpu: '0.5'
+        terminationGracePeriodSeconds: 1
+    graph:
+      children: []
+      endpoint:
+        type: REST
+      name: classifier
+      type: MODEL
+    name: example
+```
+
+This example ensures that our serving capacity does not decrease by more than 10%.
diff --git a/doc/source/index.rst b/doc/source/index.rst
@@ -74,6 +74,7 @@ Documentation Index
    Payload Logging with ELK <analytics/logging.md>
    Distributed Tracing with Jaeger <graph/distributed-tracing.md>
    Replica Scaling  <graph/scaling.md>
+   Budgeting Disruptions <graph/disruption-budgets.md>
    Custom Inference Servers <servers/custom.md>
 
 .. toctree::
@@ -101,18 +102,6 @@ Documentation Index
    Istio Ingress <ingress/istio.md>
    OpenShift <ingress/openshift.md>
 
-.. toctree::
-   :maxdepth: 1
-   :caption: Production
-
-   Supported API Protocols <graph/protocols.md>
-   CI/CD MLOps at Scale <analytics/cicd-mlops.md>
-   Metrics with Prometheus <analytics/analytics.md>
-   Payload Logging with ELK <analytics/logging.md>
-   Distributed Tracing with Jaeger <graph/distributed-tracing.md>
-   Replica Scaling  <graph/scaling.md>
-   Custom Inference Servers <servers/custom.md>
-
 .. toctree::
    :maxdepth: 1
    :caption: Streaming and Batch Processing

diff --git a/operator/apis/machinelearning.seldon.io/v1/seldondeployment_types.go b/operator/apis/machinelearning.seldon.io/v1/seldondeployment_types.go
@@ -25,6 +25,7 @@ import (
 	autoscalingv2beta2 "k8s.io/api/autoscaling/v2beta1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
 const (
@@ -289,6 +290,7 @@ type SeldonPodSpec struct {
 	Spec     v1.PodSpec        `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
 	HpaSpec  *SeldonHpaSpec    `json:"hpaSpec,omitempty" protobuf:"bytes,3,opt,name=hpaSpec"`
 	Replicas *int32            `json:"replicas,omitempty" protobuf:"bytes,4,opt,name=replicas"`
+	PdbSpec  *SeldonPdbSpec    `json:"pdbSpec,omitempty" protobuf:"bytes,5,opt,name=pdbSpec"`
 }
 
 type SeldonHpaSpec struct {
@@ -297,6 +299,23 @@ type SeldonHpaSpec struct {
 	Metrics     []autoscalingv2beta2.MetricSpec `json:"metrics,omitempty" protobuf:"bytes,3,opt,name=metrics"`
 }
 
+type SeldonPdbSpec struct {
+	// An eviction is allowed if at least "minAvailable" pods in the deployment
+	// corresponding to a componentSpec will still be available after the eviction, i.e. even in the
+	// absence of the evicted pod.  So for example you can prevent all voluntary
+	// evictions by specifying "100%".
+	// +optional
+	MinAvailable *intstr.IntOrString `json:"minAvailable,omitempty" protobuf:"bytes,1,opt,name=minAvailable"`
+
+	// An eviction is allowed if at most "maxUnavailable" pods in the deployment
+	// corresponding to a componentSpec are unavailable after the eviction, i.e. even in absence of
+	// the evicted pod. For example, one can prevent all voluntary evictions
+	// by specifying 0.
+	// MaxUnavailable and MinAvailable are mutually exclusive.
+	// +optional
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty" protobuf:"bytes,2,opt,name=maxUnavailable"`
+}
+
 type PredictiveUnitType string
 
 const (

diff --git a/operator/apis/machinelearning.seldon.io/v1/zz_generated.deepcopy.go b/operator/apis/machinelearning.seldon.io/v1/zz_generated.deepcopy.go
diff --git a/operator/config/crd/bases/machinelearning.seldon.io_seldondeployments.yaml b/operator/config/crd/bases/machinelearning.seldon.io_seldondeployments.yaml
@@ -399,6 +399,31 @@ spec:
                           type: object
                         metadata:
                           type: object
+                        pdbSpec:
+                          properties:
+                            maxUnavailable:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              description: An eviction is allowed if at most "maxUnavailable"
+                                pods in the deployment corresponding to a componentSpec
+                                are unavailable after the eviction, i.e. even in absence
+                                of the evicted pod. For example, one can prevent all
+                                voluntary evictions by specifying 0. MaxUnavailable
+                                and MinAvailable are mutually exclusive.
+                              x-kubernetes-int-or-string: true
+                            minAvailable:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              description: An eviction is allowed if at least "minAvailable"
+                                pods in the deployment corresponding to a componentSpec
+                                will still be available after the eviction, i.e. even
+                                in the absence of the evicted pod.  So for example
+                                you can prevent all voluntary evictions by specifying
+                                "100%".
+                              x-kubernetes-int-or-string: true
+                          type: object
                         replicas:
                           format: int32
                           type: integer