Add cloud sql and gcs connection for pipeline-lite deployment (#1910)

* restructure

* working example

* working example

* move mysql

* moving minio and mysql out

* add gcp

* add files

* fix test

manifests/kustomize/README.md

@@ -29,15 +29,12 @@ To get latest kubectl, visit [here](https://kubernetes.io/docs/tasks/tools/insta
 
 ## Change deploy namespace
 To deploy Kubeflow Pipelines in namespace FOO,
-- Edit [kustomization.yaml](namespaced-install/kustomization.yaml) namespace section to FOO
+- Edit [kustomization.yaml](env/dev/kustomization.yaml) namespace section to FOO
 - Then run 
 ```
-kubectl kustomize . | kubectl apply -f -
+kubectl kustomize env/dev | kubectl apply -f -
 ```
 
-## Reinstall with existing data
-TODO
-
 ## Disable the public endpoint
 By default, the deployment install an [invert proxy agent](https://github.com/google/inverting-proxy) that exposes a public URL. If you want to skip installing it,
 - Comment out the proxy component in the [kustomization.yaml](base/kustomization.yaml).
@@ -62,7 +59,7 @@ kubectl delete -f https://raw.githubusercontent.com/kubeflow/pipelines/$PIPELINE
 
 Or if you deploy through kustomize
 ```
-kubectl kustomize . | kubectl delete -f -
+kubectl kustomize env/dev | kubectl delete -f -
 ```
 # FAQ
 If sample code requires a "user-gcp-sa" secret, you could create one by 

manifests/kustomize/base/argo/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
+- minio-artifact-secret.yaml
 - workflow-controller-configmap.yaml
 - workflow-controller-deployment.yaml
 - workflow-controller-role.yaml

manifests/kustomize/base/kustomization.yaml

@@ -4,18 +4,12 @@ kind: Kustomization
 bases:
 - argo
 - crds
-- minio
-- mysql
 - pipeline
 - proxy
 
 images:
 - name: argoproj/workflow-controller
   newTag: v2.3.0
-- name: minio/minio
-  newTag: RELEASE.2018-02-09T22-40-05Z
-- name: mysql
-  newTag: "5.6"
 - name: gcr.io/ml-pipeline/api-server
   newTag: 0.1.26
 - name: gcr.io/ml-pipeline/persistenceagent

manifests/kustomize/env/dev/kustomization.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+  - ../../namespaced
+  - minio
+  - mysql
+
+# Replace with your namespace
+namespace: kubeflow
+
+images:
+  - name: mysql
+    newTag: "5.6"
+  - name: minio/minio
+    newTag: RELEASE.2018-02-09T22-40-05Z

manifests/kustomize/base/minio/kustomization.yaml → manifests/kustomize/env/dev/minio/kustomization.yaml

@@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- minio-artifact-secret.yaml
 - minio-deployment.yaml
 - minio-pvc.yaml
 - minio-service.yaml

manifests/kustomize/base/minio/minio-deployment.yaml → manifests/kustomize/env/dev/minio/minio-deployment.yaml

@@ -22,7 +22,7 @@ spec:
           value: minio
         - name: MINIO_SECRET_KEY
           value: minio123
-        image: gcr.io/ml-pipeline/minio:RELEASE.2018-02-09T22-40-05Z
+        image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z
         name: minio
         ports:
         - containerPort: 9000

manifests/kustomize/env/gcp/.gitignore

@@ -0,0 +1,2 @@
+# Ignore the GCP service account ADC file
+application_default_credentials.json

manifests/kustomize/env/gcp/kustomization.yaml

@@ -0,0 +1,26 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+  - ../../namespaced
+  - minio
+  - mysql
+
+# Replace with your namespace
+namespace: kubeflow
+
+patchesStrategicMerge:
+  - ml-pipeline-apiserver-deployment-patch.yaml
+
+images:
+  - name: gcr.io/cloudsql-docker/gce-proxy
+    newTag: "1.14"
+  - name: minio/minio
+    newTag: RELEASE.2019-08-14T20-37-41Z
+
+secretGenerator:
+  - name: user-gcp-sa
+    files:
+      # Create a service account key and stored as application_default_credentials.json in the same folder.
+      # https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys
+      - application_default_credentials.json

manifests/kustomize/env/gcp/minio/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- minio-gcs-gateway-deployment.yaml
+- minio-gcs-gateway-service.yaml

manifests/kustomize/env/gcp/minio/minio-gcs-gateway-deployment.yaml

@@ -0,0 +1,37 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: minio-deployment
+spec:
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      containers:
+        - name: minio
+          image: minio/minio:RELEASE.2019-08-14T20-37-41Z
+          args:
+            - gateway
+            - gcs
+            # Replace this with your own GCP project
+            - yang-experiment-6
+          env:
+            - name: MINIO_ACCESS_KEY
+              value: "minio"
+            - name: MINIO_SECRET_KEY
+              value: "minio123"
+            - name: GOOGLE_APPLICATION_CREDENTIALS
+              value: "/etc/credentials/application_default_credentials.json"
+          ports:
+            - containerPort: 9000
+          volumeMounts:
+            - name: gcp-sa-token
+              mountPath: "/etc/credentials"
+              readOnly: true
+      volumes:
+        - name: gcp-sa-token
+          secret:
+            secretName: user-gcp-sa

manifests/kustomize/env/gcp/minio/minio-gcs-gateway-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-service
+spec:
+  ports:
+    - port: 9000
+      targetPort: 9000
+      protocol: TCP
+  selector:
+    app: minio

manifests/kustomize/env/gcp/ml-pipeline-apiserver-deployment-patch.yaml

@@ -0,0 +1,16 @@
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: ml-pipeline
+spec:
+  template:
+    spec:
+      containers:
+        - name: ml-pipeline-api-server
+          env:
+            - name: OBJECTSTORECONFIG_BUCKETNAME
+              # Replace with your own bucket name
+              value: 'yang-experiment-6-mlpipeline'
+            - name: DBCONFIG_PASSWORD
+              # Replace with your own CloudSQL password
+              value: '123'

manifests/kustomize/env/gcp/mysql/cloudsql-proxy-deployment.yaml

@@ -0,0 +1,42 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: cloudsqlproxy
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: cloudsqlproxy
+    spec:
+      containers:
+        - image: gcr.io/cloudsql-docker/gce-proxy:1.14
+          name: cloudsqlproxy
+          command: ["/cloud_sql_proxy",
+                    "-dir=/cloudsql",
+                    # Replace with your own CloudSQL instance ID
+                    "-instances=yang-experiment-6:us-central1:kfp-test=tcp:0.0.0.0:3306",
+                    "-credential_file=/credentials/application_default_credentials.json",
+                    "term_timeout=10s"]
+          # set term_timeout if require graceful handling of shutdown
+          # NOTE: proxy will stop accepting new connections; only wait on existing connections
+          lifecycle:
+            preStop:
+              exec:
+                # (optional) add a preStop hook so that termination is delayed
+                # this is required if your server still require new connections (e.g., connection pools)
+                command: ['sleep', '10']
+          ports:
+            - name: port-database1
+              containerPort: 3306
+          volumeMounts:
+            - mountPath: /cloudsql
+              name: cloudsql
+            - mountPath: /credentials
+              name: gcp-sa-token
+      volumes:
+        - name: cloudsql
+          emptyDir:
+        - name: gcp-sa-token
+          secret:
+            secretName: user-gcp-sa

manifests/kustomize/env/gcp/mysql/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- cloudsql-proxy-deployment.yaml
+- mysql-service.yaml

manifests/kustomize/env/gcp/mysql/mysql-service.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: mysql
+spec:
+  ports:
+    - port: 3306
+      targetPort: port-database1
+  selector:
+    app: cloudsqlproxy

manifests/kustomize/namespaced-install/kustomization.yaml → manifests/kustomize/namespaced/kustomization.yaml

@@ -2,13 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 bases:
-- ../base
+  - ../base
 
 resources:
 - namespace.yaml
 
-namespace: kubeflow
-
 patchesStrategicMerge:
 - workflow-controller-configmap.yaml
 - ml-pipeline-persistenceagent-deployment-patch.yaml

test/manifests/kustomization.yaml

@@ -4,4 +4,4 @@ kind: Kustomization
 # Actual image overrides will be added in test scripts.
 images: []
 resources:
-- ../../manifests/kustomize/namespaced-install
+- ../../manifests/kustomize/env/dev