Pipeline Persistence (#188)

* initial pipeline db commit

* add loadbalancer for pipelines and rebalance

* allow modelgateway to do retries

* Makefile fixes for host

* ensure pipeline updates sent

* test on k8s

* lint

* check for db

* review comments

* review comments

* review fixes

Makefile

@@ -20,10 +20,9 @@ deploy-k8s:
 
 .PHONY: undeploy-k8s
 undeploy-k8s:
-	kubectl delete -f k8s/yaml/seldon-v2-servers.yaml
-	kubectl delete -f k8s/yaml/seldon-v2-components.yaml
-	kubectl delete -f k8s/yaml/seldon-v2-crds.yaml
-
+	kubectl delete --ignore-not-found=true -f k8s/seldon-v2-servers.yaml
+	kubectl delete --ignore-not-found=true -f k8s/seldon-v2-components.yaml
+	kubectl delete --ignore-not-found=true -f k8s/seldon-v2-crds.yaml
 
 #
 # Dev

apis/Makefile

@@ -15,6 +15,7 @@ build:
 		./mlops/agent_debug/agent_debug.proto \
 		./mlops/proxy/proxy.proto \
 		./mlops/scheduler/scheduler.proto \
+		./mlops/scheduler/storage.proto \
 		./mlops/chainer/chainer.proto \
 		./mlops/v2_dataplane/v2_dataplane.proto
 

apis/mlops/scheduler/scheduler.proto

@@ -246,9 +246,11 @@ message ExperimentStatusRequest {
 
 message Pipeline {
   string name = 1;
-  repeated PipelineStep steps = 2;
-  optional PipelineOutput output = 3;
-  optional KubernetesMeta kubernetesMeta = 4;
+  string uid = 2;
+  uint32 version = 3;
+  repeated PipelineStep steps = 4;
+  optional PipelineOutput output = 5;
+  optional KubernetesMeta kubernetesMeta = 6;
 }
 
 message PipelineStep {

apis/mlops/scheduler/storage.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+
+package seldon.mlops.scheduler;
+
+option go_package = "github.com/seldonio/seldon-core/scheduler/apis/mlops/scheduler";
+
+import "mlops/scheduler/scheduler.proto";
+
+message PipelineSnapshot {
+  string name = 1;
+  uint32 lastVersion = 2;
+  repeated PipelineWithState versions = 3;
+  bool deleted = 4;
+}

k8s/helm-charts/seldon-core-v2-setup/templates/seldon-v2-components.yaml

@@ -771,6 +771,7 @@ spec:
       - args:
         - --http-port=9010
         - --grpc-port=9011
+        - --metrics-port=9006
         - --config-path=/mnt/config/kafka.json
         command:
         - /bin/pipelinegateway
@@ -788,6 +789,9 @@ spec:
         - containerPort: 9011
           name: grpc
           protocol: TCP
+        - containerPort: 9006
+          name: metrics
+          protocol: TCP
         resources:
           requests:
             cpu: '{{ .Values.pipelinegateway.resources.requests.cpu }}'
@@ -805,7 +809,7 @@ spec:
         name: config-volume
 ---
 apiVersion: apps/v1
-kind: Deployment
+kind: StatefulSet
 metadata:
   labels:
     control-plane: seldon-scheduler
@@ -816,6 +820,7 @@ spec:
   selector:
     matchLabels:
       control-plane: seldon-scheduler
+  serviceName: seldon-scheduler
   template:
     metadata:
       labels:
@@ -824,6 +829,7 @@ spec:
       containers:
       - args:
         - --pipeline-gateway-host=seldon-pipelinegateway
+        - --pipeline-db-path=/mnt/scheduler/pipelinedb
         command:
         - /bin/scheduler
         image: seldonio/seldon-scheduler:latest
@@ -842,10 +848,22 @@ spec:
           requests:
             cpu: 100m
             memory: 200Mi
+        volumeMounts:
+        - mountPath: /mnt/scheduler
+          name: scheduler-state
       securityContext:
         runAsUser: 8888
       serviceAccountName: seldon-scheduler
       terminationGracePeriodSeconds: 5
+  volumeClaimTemplates:
+  - metadata:
+      name: scheduler-state
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1Gi
 ---
 apiVersion: mlops.seldon.io/v1alpha1
 kind: ServerConfig