From 06b7ad659f15b76e2722105b66f6195abe032847 Mon Sep 17 00:00:00 2001 From: IronPan Date: Thu, 22 Aug 2019 10:03:28 -0700 Subject: [PATCH] Cleanup pipeline-lite deployment (#1921) * restructure * working example * working example * move mysql * moving minio and mysql out * add gcp * add files * fix test * extract parameters to single place * update * update readme * update readme * address pr comment --- manifests/kustomize/README.md | 53 ++++++++++++------- ...-pipeline-persistenceagent-deployment.yaml | 7 ++- ...pipeline-scheduledworkflow-deployment.yaml | 5 ++ .../pipeline/ml-pipeline-ui-deployment.yaml | 5 ++ .../ml-pipeline-viewer-crd-deployment.yaml | 4 ++ manifests/kustomize/env/gcp/README.md | 13 +++++ .../env/gcp/gcp-configurations-patch.yaml | 34 ++++++++++++ .../kustomize/env/gcp/kustomization.yaml | 4 +- .../minio/minio-gcs-gateway-deployment.yaml | 11 ++-- ...l-pipeline-apiserver-deployment-patch.yaml | 16 ------ .../gcp/mysql/cloudsql-proxy-deployment.yaml | 14 ++++- manifests/kustomize/namespaced-install.yaml | 20 ++++--- .../kustomize/namespaced/kustomization.yaml | 4 -- ...ine-persistenceagent-deployment-patch.yaml | 12 ----- ...ne-scheduledworkflow-deployment-patch.yaml | 12 ----- .../ml-pipeline-ui-deployment-patch.yaml | 12 ----- ...-pipeline-viewer-crd-deployment-patch.yaml | 12 ----- 17 files changed, 135 insertions(+), 103 deletions(-) create mode 100644 manifests/kustomize/env/gcp/README.md create mode 100644 manifests/kustomize/env/gcp/gcp-configurations-patch.yaml delete mode 100644 manifests/kustomize/env/gcp/ml-pipeline-apiserver-deployment-patch.yaml delete mode 100644 manifests/kustomize/namespaced/ml-pipeline-persistenceagent-deployment-patch.yaml delete mode 100644 manifests/kustomize/namespaced/ml-pipeline-scheduledworkflow-deployment-patch.yaml delete mode 100644 manifests/kustomize/namespaced/ml-pipeline-ui-deployment-patch.yaml delete mode 100644 manifests/kustomize/namespaced/ml-pipeline-viewer-crd-deployment-patch.yaml diff --git a/manifests/kustomize/README.md b/manifests/kustomize/README.md index 72cb24f31fd..3595f90a459 100644 --- a/manifests/kustomize/README.md +++ b/manifests/kustomize/README.md @@ -1,41 +1,40 @@ +# Install Kubeflow Pipelines This folder contains Kubeflow Pipelines Kustomize manifests for a light weight deployment. You can follow the instruction and deploy Kubeflow Pipelines in an existing cluster. -# TL;DR +## TL;DR -If you want to skip any customization, you can deploy Kubeflow Pipelines by running +Deploy latest version of Kubeflow Pipelines ``` export PIPELINE_VERSION=0.1.26 kubectl apply -f https://raw.githubusercontent.com/kubeflow/pipelines/$PIPELINE_VERSION/manifests/kustomize/namespaced-install.yaml ``` -You might lack the permission to create role and command might partially fail. If so, bind your account as cluster admin and rerun the same command. -(Or role creator in your namespace) +Then get the Pipeline URL ``` -kubectl create clusterrolebinding your-binding --clusterrole=cluster-admin --user=[your-user-name] -``` - -When deployment is complete, you can access Kubeflow Pipelines UI by an IAM controlled public endpoint, which can be found by -``` -kubectl describe configmap inverse-proxy-config -n kubeflow +kubectl describe configmap inverse-proxy-config -n kubeflow | grep googleusercontent.com ``` -and check the Hostname section. The endpoint should have format like **1234567-dot-datalab-vm-us-west1.googleusercontent.com** -# Customization +## Customization Customization can be done through Kustomize [Overlay](https://github.com/kubernetes-sigs/kustomize/blob/master/docs/glossary.md#overlay). Note - The instruction below assume you installed kubectl v1.14.0 or later, which has native support of kustomize. To get latest kubectl, visit [here](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -## Change deploy namespace +### Deploy on GCP with CloudSQL and GCS +See [here](env/gcp/README.md) for more details. + +### Change deploy namespace To deploy Kubeflow Pipelines in namespace FOO, -- Edit [kustomization.yaml](env/dev/kustomization.yaml) namespace section to FOO +- Edit [dev/kustomization.yaml](env/dev/kustomization.yaml) or [gcp/kustomization.yaml](env/gcp/kustomization.yaml) namespace section to FOO - Then run ``` kubectl kustomize env/dev | kubectl apply -f - +# or +kubectl kustomize env/gcp | kubectl apply -f - ``` -## Disable the public endpoint +### Disable the public endpoint By default, the deployment install an [invert proxy agent](https://github.com/google/inverting-proxy) that exposes a public URL. If you want to skip installing it, - Comment out the proxy component in the [kustomization.yaml](base/kustomization.yaml). - Then run @@ -51,20 +50,36 @@ and open http://localhost:8080/ -# Uninstall +## Uninstall You can uninstall Kubeflow Pipelines by running ``` +export PIPELINE_VERSION=0.1.26 kubectl delete -f https://raw.githubusercontent.com/kubeflow/pipelines/$PIPELINE_VERSION/manifests/kustomize/namespaced-install.yaml ``` Or if you deploy through kustomize ``` kubectl kustomize env/dev | kubectl delete -f - +# or +kubectl kustomize env/gcp | kubectl delete -f - +``` + +## Troubleshooting + +### Permission error installing Kubeflow Pipelines to a cluster +Run +``` +kubectl create clusterrolebinding your-binding --clusterrole=cluster-admin --user=[your-user-name] ``` -# FAQ + +### Samples requires "user-gcp-sa" secret If sample code requires a "user-gcp-sa" secret, you could create one by -- First download the GCE VM service account token following this [instruction](https://cloud.google.com/kubernetes-engine/docs/tutorials/authenticating-to-cloud-platform#step_3_create_service_account_credentials) +- First download the GCE VM service account token [Document](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) +``` +gcloud iam service-accounts keys create application_default_credentials.json \ + --iam-account [SA-NAME]@[PROJECT-ID].iam.gserviceaccount.com +``` - Run ``` -kubectl create secret -n [your-namespace] generic user-gcp-sa --from-file=user-gcp-sa.json=[your-token-file].json +kubectl create secret -n [your-namespace] generic user-gcp-sa --from-file=user-gcp-sa.json=application_default_credentials.json ``` diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml index ef267e141d1..a6341f30220 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-persistenceagent-deployment.yaml @@ -14,7 +14,12 @@ spec: app: ml-pipeline-persistenceagent spec: containers: - - image: gcr.io/ml-pipeline/persistenceagent:0.1.15 + - env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: gcr.io/ml-pipeline/persistenceagent:0.1.15 imagePullPolicy: IfNotPresent name: ml-pipeline-persistenceagent serviceAccountName: ml-pipeline-persistenceagent \ No newline at end of file diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml index acb7a0b1ef4..f896bd2beee 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-scheduledworkflow-deployment.yaml @@ -17,4 +17,9 @@ spec: - image: gcr.io/ml-pipeline/scheduledworkflow:0.1.15 imagePullPolicy: IfNotPresent name: ml-pipeline-scheduledworkflow + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace serviceAccountName: ml-pipeline-scheduledworkflow \ No newline at end of file diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml index 2078073c033..d198883bd3a 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-ui-deployment.yaml @@ -19,4 +19,9 @@ spec: name: ml-pipeline-ui ports: - containerPort: 3000 + env: + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace serviceAccountName: ml-pipeline-ui \ No newline at end of file diff --git a/manifests/kustomize/base/pipeline/ml-pipeline-viewer-crd-deployment.yaml b/manifests/kustomize/base/pipeline/ml-pipeline-viewer-crd-deployment.yaml index 3163d0dee87..6a0d4787543 100644 --- a/manifests/kustomize/base/pipeline/ml-pipeline-viewer-crd-deployment.yaml +++ b/manifests/kustomize/base/pipeline/ml-pipeline-viewer-crd-deployment.yaml @@ -20,4 +20,8 @@ spec: env: - name: MAX_NUM_VIEWERS value: "50" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace serviceAccountName: ml-pipeline-viewer-crd-service-account diff --git a/manifests/kustomize/env/gcp/README.md b/manifests/kustomize/env/gcp/README.md new file mode 100644 index 00000000000..99f7a1c83d4 --- /dev/null +++ b/manifests/kustomize/env/gcp/README.md @@ -0,0 +1,13 @@ +# TL;DR +1. To access the GCP services, the application needs a GCP service account token. Download the token to the current folder manifests/kustomize/env/gcp. [Reference](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) +``` +gcloud iam service-accounts keys create application_default_credentials.json \ + --iam-account [SA-NAME]@[PROJECT-ID].iam.gserviceaccount.com +``` +2. [Create](https://cloud.google.com/sql/docs/mysql/quickstart) or use an existing CloudSQL instance. The service account should have the access to the CloudSQL instance. +3. Fill in gcp-configurations-patch.yaml with your CloudSQL and GCS configuration. + +# Why Cloud SQL and GCS +Kubeflow Pipelines keeps its metadata in mysql database and artifacts in S3 compatible object storage. +Using CloudSQL and GCS for persisting the data provides better reliability and performance, as well as things like data backups, and usage monitoring. +This is the recommended setup especially for production environments. diff --git a/manifests/kustomize/env/gcp/gcp-configurations-patch.yaml b/manifests/kustomize/env/gcp/gcp-configurations-patch.yaml new file mode 100644 index 00000000000..22092944b1b --- /dev/null +++ b/manifests/kustomize/env/gcp/gcp-configurations-patch.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: ml-pipeline +spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: OBJECTSTORECONFIG_BUCKETNAME + # Don't add gs:// prefix. + # If bucket doesn't exist, the deployment will create one. + value: '' + - name: DBCONFIG_PASSWORD + value: '' +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: cloudsqlproxy +spec: + template: + spec: + containers: + - name: cloudsqlproxy + env: + - name: GCP_PROJECT + value: '' + - name: CLOUDSQL_ZONE + # E.g. us-central1 + value: '' + - name: CLOUDSQL_INSTANCE_NAME + value: '' \ No newline at end of file diff --git a/manifests/kustomize/env/gcp/kustomization.yaml b/manifests/kustomize/env/gcp/kustomization.yaml index 5345ff2852f..eb9d06c5529 100644 --- a/manifests/kustomize/env/gcp/kustomization.yaml +++ b/manifests/kustomize/env/gcp/kustomization.yaml @@ -10,12 +10,12 @@ bases: namespace: kubeflow patchesStrategicMerge: - - ml-pipeline-apiserver-deployment-patch.yaml + - gcp-configurations-patch.yaml images: - name: gcr.io/cloudsql-docker/gce-proxy newTag: "1.14" - - name: minio/minio + - name: gcr.io/ml-pipeline/minio newTag: RELEASE.2019-08-14T20-37-41Z secretGenerator: diff --git a/manifests/kustomize/env/gcp/minio/minio-gcs-gateway-deployment.yaml b/manifests/kustomize/env/gcp/minio/minio-gcs-gateway-deployment.yaml index 67711e43f3f..28866484ec2 100644 --- a/manifests/kustomize/env/gcp/minio/minio-gcs-gateway-deployment.yaml +++ b/manifests/kustomize/env/gcp/minio/minio-gcs-gateway-deployment.yaml @@ -1,8 +1,11 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1beta2 kind: Deployment metadata: - name: minio-deployment + name: minio spec: + selector: + matchLabels: + app: minio strategy: type: Recreate template: @@ -12,12 +15,10 @@ spec: spec: containers: - name: minio - image: minio/minio:RELEASE.2019-08-14T20-37-41Z + image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z args: - gateway - gcs - # Replace this with your own GCP project - - yang-experiment-6 env: - name: MINIO_ACCESS_KEY value: "minio" diff --git a/manifests/kustomize/env/gcp/ml-pipeline-apiserver-deployment-patch.yaml b/manifests/kustomize/env/gcp/ml-pipeline-apiserver-deployment-patch.yaml deleted file mode 100644 index 1400971e350..00000000000 --- a/manifests/kustomize/env/gcp/ml-pipeline-apiserver-deployment-patch.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: ml-pipeline -spec: - template: - spec: - containers: - - name: ml-pipeline-api-server - env: - - name: OBJECTSTORECONFIG_BUCKETNAME - # Replace with your own bucket name - value: 'yang-experiment-6-mlpipeline' - - name: DBCONFIG_PASSWORD - # Replace with your own CloudSQL password - value: '123' diff --git a/manifests/kustomize/env/gcp/mysql/cloudsql-proxy-deployment.yaml b/manifests/kustomize/env/gcp/mysql/cloudsql-proxy-deployment.yaml index d0a5bec0ccd..dfd3854e720 100644 --- a/manifests/kustomize/env/gcp/mysql/cloudsql-proxy-deployment.yaml +++ b/manifests/kustomize/env/gcp/mysql/cloudsql-proxy-deployment.yaml @@ -1,8 +1,11 @@ -apiVersion: extensions/v1beta1 +apiVersion: apps/v1beta2 kind: Deployment metadata: name: cloudsqlproxy spec: + selector: + matchLabels: + app: cloudsqlproxy replicas: 1 template: metadata: @@ -12,10 +15,17 @@ spec: containers: - image: gcr.io/cloudsql-docker/gce-proxy:1.14 name: cloudsqlproxy + env: + - name: GCP_PROJECT + value: "" + - name: CLOUDSQL_ZONE + value: "" + - name: CLOUDSQL_INSTANCE_NAME + value: "" command: ["/cloud_sql_proxy", "-dir=/cloudsql", # Replace with your own CloudSQL instance ID - "-instances=yang-experiment-6:us-central1:kfp-test=tcp:0.0.0.0:3306", + "-instances=$(GCP_PROJECT):$(CLOUDSQL_ZONE):$(CLOUDSQL_INSTANCE_NAME)=tcp:0.0.0.0:3306", "-credential_file=/credentials/application_default_credentials.json", "term_timeout=10s"] # set term_timeout if require graceful handling of shutdown diff --git a/manifests/kustomize/namespaced-install.yaml b/manifests/kustomize/namespaced-install.yaml index 93e585cee4b..2f8d7e095c9 100644 --- a/manifests/kustomize/namespaced-install.yaml +++ b/manifests/kustomize/namespaced-install.yaml @@ -637,7 +637,7 @@ spec: value: minio - name: MINIO_SECRET_KEY value: minio123 - image: gcr.io/ml-pipeline/minio:RELEASE.2018-02-09T22-40-05Z + image: gcr.io/ml-pipeline/minio:RELEASE.2019-08-14T20-37-41Z name: minio ports: - containerPort: 9000 @@ -669,7 +669,9 @@ spec: containers: - env: - name: NAMESPACE - value: kubeflow + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: gcr.io/ml-pipeline/persistenceagent:0.1.26 imagePullPolicy: IfNotPresent name: ml-pipeline-persistenceagent @@ -694,7 +696,9 @@ spec: containers: - env: - name: NAMESPACE - value: kubeflow + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: gcr.io/ml-pipeline/scheduledworkflow:0.1.26 imagePullPolicy: IfNotPresent name: ml-pipeline-scheduledworkflow @@ -719,7 +723,9 @@ spec: containers: - env: - name: MINIO_NAMESPACE - value: kubeflow + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: gcr.io/ml-pipeline/frontend:0.1.26 imagePullPolicy: IfNotPresent name: ml-pipeline-ui @@ -745,10 +751,12 @@ spec: spec: containers: - env: - - name: NAMESPACE - value: kubeflow - name: MAX_NUM_VIEWERS value: "50" + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: gcr.io/ml-pipeline/viewer-crd-controller:0.1.26 imagePullPolicy: Always name: ml-pipeline-viewer-crd diff --git a/manifests/kustomize/namespaced/kustomization.yaml b/manifests/kustomize/namespaced/kustomization.yaml index 7f04d68c127..1bd947898d1 100644 --- a/manifests/kustomize/namespaced/kustomization.yaml +++ b/manifests/kustomize/namespaced/kustomization.yaml @@ -9,10 +9,6 @@ resources: patchesStrategicMerge: - workflow-controller-configmap.yaml -- ml-pipeline-persistenceagent-deployment-patch.yaml -- ml-pipeline-scheduledworkflow-deployment-patch.yaml -- ml-pipeline-viewer-crd-deployment-patch.yaml -- ml-pipeline-ui-deployment-patch.yaml vars: - name: NAMESPACE diff --git a/manifests/kustomize/namespaced/ml-pipeline-persistenceagent-deployment-patch.yaml b/manifests/kustomize/namespaced/ml-pipeline-persistenceagent-deployment-patch.yaml deleted file mode 100644 index de706660723..00000000000 --- a/manifests/kustomize/namespaced/ml-pipeline-persistenceagent-deployment-patch.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: ml-pipeline-persistenceagent -spec: - template: - spec: - containers: - - name: ml-pipeline-persistenceagent - env: - - name: NAMESPACE - value: $(NAMESPACE) diff --git a/manifests/kustomize/namespaced/ml-pipeline-scheduledworkflow-deployment-patch.yaml b/manifests/kustomize/namespaced/ml-pipeline-scheduledworkflow-deployment-patch.yaml deleted file mode 100644 index 6837ff171c6..00000000000 --- a/manifests/kustomize/namespaced/ml-pipeline-scheduledworkflow-deployment-patch.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: ml-pipeline-scheduledworkflow -spec: - template: - spec: - containers: - - name: ml-pipeline-scheduledworkflow - env: - - name: NAMESPACE - value: $(NAMESPACE) diff --git a/manifests/kustomize/namespaced/ml-pipeline-ui-deployment-patch.yaml b/manifests/kustomize/namespaced/ml-pipeline-ui-deployment-patch.yaml deleted file mode 100644 index 52adfebdb86..00000000000 --- a/manifests/kustomize/namespaced/ml-pipeline-ui-deployment-patch.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: ml-pipeline-ui -spec: - template: - spec: - containers: - - name: ml-pipeline-ui - env: - - name: MINIO_NAMESPACE - value: $(NAMESPACE) diff --git a/manifests/kustomize/namespaced/ml-pipeline-viewer-crd-deployment-patch.yaml b/manifests/kustomize/namespaced/ml-pipeline-viewer-crd-deployment-patch.yaml deleted file mode 100644 index 848134f1fbc..00000000000 --- a/manifests/kustomize/namespaced/ml-pipeline-viewer-crd-deployment-patch.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: ml-pipeline-viewer-crd -spec: - template: - spec: - containers: - - name: ml-pipeline-viewer-crd - env: - - name: NAMESPACE - value: $(NAMESPACE)