From 95485d900c558f284d1b6840cbfef30d58fb6274 Mon Sep 17 00:00:00 2001 From: Michael Meredith Date: Tue, 9 Nov 2021 21:31:24 -0500 Subject: [PATCH 1/8] update split_resources python file --- ...deployments.machinelearning.seldon.io.yaml | 5 +- ...deployments.machinelearning.seldon.io.yaml | 5 +- .../deployment_seldon-controller-manager.yaml | 3 ++ operator/helm/create_templates.sh | 4 +- operator/helm/split_resources.py | 51 +++++++++++++++---- 5 files changed, 54 insertions(+), 14 deletions(-) diff --git a/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml b/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml index 338d082d87..4d79331f74 100644 --- a/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml +++ b/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml @@ -5,7 +5,10 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: '{{ include "seldon.namespace" . }}/seldon-serving-cert' - controller-gen.kubebuilder.io/version: v0.5.0 + controller-gen.kubebuilder.io/version: v0.4.1 +{{- with .Values.crd.annotations }} +{{- toYaml . | nindent 4}} +{{- end }} labels: app: seldon app.kubernetes.io/instance: '{{ .Release.Name }}' diff --git a/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml b/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml index b957c17224..402bcaa9e2 100644 --- a/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml +++ b/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml @@ -5,7 +5,10 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: '{{ include "seldon.namespace" . }}/seldon-serving-cert' - controller-gen.kubebuilder.io/version: v0.5.0 + controller-gen.kubebuilder.io/version: v0.4.1 +{{- with .Values.crd.annotations }} +{{- toYaml . | nindent 4}} +{{- end }} name: seldondeployments.machinelearning.seldon.io spec: group: machinelearning.seldon.io diff --git a/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml b/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml index 73f96e82f9..ba328c9f93 100644 --- a/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml +++ b/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml @@ -21,6 +21,9 @@ spec: template: metadata: annotations: +{{- with .Values.manager.annotations }} +{{- toYaml . | nindent 4}} +{{- end }} prometheus.io/scrape: 'true' sidecar.istio.io/inject: 'false' labels: diff --git a/operator/helm/create_templates.sh b/operator/helm/create_templates.sh index 215e3c7c75..dd7ee45efe 100755 --- a/operator/helm/create_templates.sh +++ b/operator/helm/create_templates.sh @@ -6,7 +6,7 @@ rm -rf tmp mkdir tmp cd tmp kustomize build ../../config/$1 > tt.yaml -csplit --suppress-matched tt.yaml "/^---/" "{*}" +gcsplit --suppress-matched tt.yaml "/^---/" "{*}" # manually add secret file from cert kustomize option cp ../../config/cert/secret.yaml xxSecret -python ../split_resources.py --folder ../../../helm-charts/seldon-core-operator/templates +python3 ../split_resources.py --folder ../../../helm-charts/seldon-core-operator/templates diff --git a/operator/helm/split_resources.py b/operator/helm/split_resources.py index dd1f4cd3b4..7fab9aa62b 100644 --- a/operator/helm/split_resources.py +++ b/operator/helm/split_resources.py @@ -5,7 +5,8 @@ import yaml parser = argparse.ArgumentParser() -parser.add_argument("--prefix", default="xx", help="find files matching prefix") +parser.add_argument("--prefix", default="xx", + help="find files matching prefix") parser.add_argument("--folder", required=True, help="Output folder") args, _ = parser.parse_known_args() @@ -29,6 +30,9 @@ HELM_CREATERESOURCES_RBAC_IF_START = "{{- if .Values.managerCreateResources }}\n" HELM_K8S_V1_CRD_IF_START = '{{- if or (ge (int (regexFind "[0-9]+" .Capabilities.KubeVersion.Minor)) 18) (.Values.crd.forcev1) }}\n' HELM_K8S_V1BETA1_CRD_IF_START = '{{- if or (lt (int (regexFind "[0-9]+" .Capabilities.KubeVersion.Minor)) 18) (.Values.crd.forcev1beta1) }}\n' +HELM_CRD_ANNOTATIONS_WITH_START = '{{- with .Values.crd.annotations }}\n' +HELM_ANNOTATIONS_TOYAML = '{{- toYaml . | nindent 4}}\n' +HELM_CONTROLER_DEP_ANNOTATIONS_WITH_START = '{{- with .Values.manager.annotations }}\n' HELM_IF_END = "{{- end }}\n" HELM_ENV_SUBST = { @@ -96,7 +100,8 @@ def helm_namespace_override(): webhookData + '{{- $altNames := list ( printf "seldon-webhook-service.%s" (include "seldon.namespace" .) ) ( printf "seldon-webhook-service.%s.svc" (include "seldon.namespace" .) ) -}}\n' ) - webhookData = webhookData + '{{- $ca := genCA "custom-metrics-ca" 365 -}}\n' + webhookData = webhookData + \ + '{{- $ca := genCA "custom-metrics-ca" 365 -}}\n' webhookData = ( webhookData + '{{- $cert := genSignedCert "seldon-webhook-service" nil $altNames 365 $ca -}}\n' @@ -108,7 +113,8 @@ def helm_namespace_override(): kind = res["kind"].lower() name = res["metadata"]["name"].lower() version = res["apiVersion"] - filename = args.folder + "/" + (kind + "_" + name).lower() + ".yaml" + filename = args.folder + "/" + \ + (kind + "_" + name).lower() + ".yaml" print(filename) print(version) if ( @@ -201,7 +207,8 @@ def helm_namespace_override(): if portSpec["name"] == "webhook-server": portSpec["containerPort"] = helm_value("webhook.port") for argIdx in range( - 0, len(res["spec"]["template"]["spec"]["containers"][0]["args"]) + 0, len(res["spec"]["template"]["spec"] + ["containers"][0]["args"]) ): if ( res["spec"]["template"]["spec"]["containers"][0]["args"][argIdx] @@ -216,7 +223,8 @@ def helm_namespace_override(): if kind == "configmap" and name == "seldon-config": res["data"]["credentials"] = helm_value_json("credentials") - res["data"]["predictor_servers"] = helm_value_json("predictor_servers") + res["data"]["predictor_servers"] = helm_value_json( + "predictor_servers") res["data"]["storageInitializer"] = helm_value_json( "storageInitializer" ) @@ -239,7 +247,8 @@ def helm_namespace_override(): res["roleRef"]["name"] + "-" + helm_namespace_override() ) if name == "seldon-manager-rolebinding": - res["subjects"][0]["name"] = helm_value("serviceAccount.name") + res["subjects"][0]["name"] = helm_value( + "serviceAccount.name") res["subjects"][0]["namespace"] = helm_namespace_override() elif name != "seldon-spartakus-volunteer": res["subjects"][0]["namespace"] = helm_namespace_override() @@ -252,7 +261,8 @@ def helm_namespace_override(): or name == "seldon1-manager-sas-rolebinding" or name == "seldon-leader-election-rolebinding" ): - res["subjects"][0]["name"] = helm_value("serviceAccount.name") + res["subjects"][0]["name"] = helm_value( + "serviceAccount.name") res["subjects"][0]["namespace"] = helm_namespace_override() # Update webhook certificates @@ -301,7 +311,8 @@ def helm_namespace_override(): # Update webhook service port if kind == "service" and name == "seldon-webhook-service": - res["spec"]["ports"][0]["targetPort"] = helm_value("webhook.port") + res["spec"]["ports"][0]["targetPort"] = helm_value( + "webhook.port") fdata = yaml.dump(res, width=1000) @@ -392,7 +403,13 @@ def helm_namespace_override(): fdata = ( HELM_CRD_IF_START + HELM_K8S_V1BETA1_CRD_IF_START - + fdata + + re.sub( + r"(.*controller-gen.kubebuilder.io/version.*\n)", + r"\1" + HELM_CRD_ANNOTATIONS_WITH_START + + HELM_ANNOTATIONS_TOYAML + HELM_IF_END, + fdata, + re.M, + ) + HELM_IF_END + HELM_IF_END ) @@ -403,7 +420,13 @@ def helm_namespace_override(): fdata = ( HELM_CRD_IF_START + HELM_K8S_V1_CRD_IF_START - + fdata + + re.sub( + r"(.*controller-gen.kubebuilder.io/version.*\n)", + r"\1" + HELM_CRD_ANNOTATIONS_WITH_START + + HELM_ANNOTATIONS_TOYAML + HELM_IF_END, + fdata, + re.M, + ) + HELM_IF_END + HELM_IF_END ) @@ -425,6 +448,14 @@ def helm_namespace_override(): re.M, ) + fdata = re.sub( + r"(.*template:\n.*metadata:\n.*annotations:\n)", + r"\1" + HELM_CONTROLER_DEP_ANNOTATIONS_WITH_START + + HELM_ANNOTATIONS_TOYAML + HELM_IF_END, + fdata, + re.M, + ) + # make sure webhook is not quoted as its an int fdata = fdata.replace( "'{{ .Values.webhook.port }}'", "{{ .Values.webhook.port }}" From b840d095cbec81ffda444b359315785a22031cc4 Mon Sep 17 00:00:00 2001 From: Mike Meredith Date: Thu, 4 Nov 2021 13:05:22 -0400 Subject: [PATCH 2/8] add annotations to crds --- helm-charts/seldon-core-operator/templates/_helpers.tpl | 5 +++++ helm-charts/seldon-core-operator/values.yaml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/helm-charts/seldon-core-operator/templates/_helpers.tpl b/helm-charts/seldon-core-operator/templates/_helpers.tpl index d18919c6dc..6cdbe366a1 100644 --- a/helm-charts/seldon-core-operator/templates/_helpers.tpl +++ b/helm-charts/seldon-core-operator/templates/_helpers.tpl @@ -52,3 +52,8 @@ Create the name of the service account to use {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} + + +{{- define "seldon.annotations" -}} +{{ toYaml .Values.globalAnnotations }} +{{- end -}} \ No newline at end of file diff --git a/helm-charts/seldon-core-operator/values.yaml b/helm-charts/seldon-core-operator/values.yaml index 4cf5bfaa4b..ccccf03f8c 100644 --- a/helm-charts/seldon-core-operator/values.yaml +++ b/helm-charts/seldon-core-operator/values.yaml @@ -161,6 +161,8 @@ crd: # Whether to force the use of the v1beta1 or v1 CRD. forceV1: false forceV1beta1: false + annotations: # Annotations to add to the CRD if not created by the Manager + my-annotation: my-annotation-value # Warning: credentials will be depricated soon, please use defaultEnvSecretRefName above # For more info please check the documentation From 674d573ccd7e9159b12391832e2c85003d2df3a0 Mon Sep 17 00:00:00 2001 From: Mike Meredith Date: Thu, 4 Nov 2021 13:08:22 -0400 Subject: [PATCH 3/8] add annotations to crds --- helm-charts/seldon-core-operator/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-charts/seldon-core-operator/values.yaml b/helm-charts/seldon-core-operator/values.yaml index ccccf03f8c..428ae1befc 100644 --- a/helm-charts/seldon-core-operator/values.yaml +++ b/helm-charts/seldon-core-operator/values.yaml @@ -80,6 +80,7 @@ manager: memoryRequest: 200Mi logLevel: INFO leaderElectionID: a33bd623.machinelearning.seldon.io + annotations: {} rbac: configmap: create: true @@ -161,8 +162,7 @@ crd: # Whether to force the use of the v1beta1 or v1 CRD. forceV1: false forceV1beta1: false - annotations: # Annotations to add to the CRD if not created by the Manager - my-annotation: my-annotation-value + annotations: {} # Annotations to add to the CRD if not created by the Manager # Warning: credentials will be depricated soon, please use defaultEnvSecretRefName above # For more info please check the documentation From cd6b68ca0060d7e1abc18e2744d9ec09ce5b9121 Mon Sep 17 00:00:00 2001 From: Mike Meredith Date: Thu, 4 Nov 2021 13:14:39 -0400 Subject: [PATCH 4/8] add annotations to deployment template --- helm-charts/seldon-core-operator/templates/_helpers.tpl | 5 ----- .../templates/deployment_seldon-controller-manager.yaml | 4 ++++ 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/helm-charts/seldon-core-operator/templates/_helpers.tpl b/helm-charts/seldon-core-operator/templates/_helpers.tpl index 6cdbe366a1..d18919c6dc 100644 --- a/helm-charts/seldon-core-operator/templates/_helpers.tpl +++ b/helm-charts/seldon-core-operator/templates/_helpers.tpl @@ -52,8 +52,3 @@ Create the name of the service account to use {{ default "default" .Values.serviceAccount.name }} {{- end -}} {{- end -}} - - -{{- define "seldon.annotations" -}} -{{ toYaml .Values.globalAnnotations }} -{{- end -}} \ No newline at end of file diff --git a/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml b/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml index ba328c9f93..894f170a9e 100644 --- a/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml +++ b/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml @@ -7,6 +7,10 @@ metadata: app.kubernetes.io/name: '{{ include "seldon.name" . }}' app.kubernetes.io/version: '{{ .Chart.Version }}' control-plane: seldon-controller-manager + {{with .Values.manager.annotations}} + annotations: + {{- toYaml . | nindent 4}} + {{- end }} name: seldon-controller-manager namespace: '{{ include "seldon.namespace" . }}' spec: From d48f112bfb9851c8740b5e0c44080bd02daaf0b6 Mon Sep 17 00:00:00 2001 From: Michael Meredith Date: Tue, 9 Nov 2021 21:38:43 -0500 Subject: [PATCH 5/8] merge origin --- operator/helm/create_templates.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/helm/create_templates.sh b/operator/helm/create_templates.sh index dd7ee45efe..215e3c7c75 100755 --- a/operator/helm/create_templates.sh +++ b/operator/helm/create_templates.sh @@ -6,7 +6,7 @@ rm -rf tmp mkdir tmp cd tmp kustomize build ../../config/$1 > tt.yaml -gcsplit --suppress-matched tt.yaml "/^---/" "{*}" +csplit --suppress-matched tt.yaml "/^---/" "{*}" # manually add secret file from cert kustomize option cp ../../config/cert/secret.yaml xxSecret -python3 ../split_resources.py --folder ../../../helm-charts/seldon-core-operator/templates +python ../split_resources.py --folder ../../../helm-charts/seldon-core-operator/templates From e196c82f3bf6a0404a45b02adbbaf0e1dafe0107 Mon Sep 17 00:00:00 2001 From: Michael Meredith Date: Tue, 9 Nov 2021 21:43:45 -0500 Subject: [PATCH 6/8] update chart readme --- helm-charts/seldon-core-operator/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm-charts/seldon-core-operator/README.md b/helm-charts/seldon-core-operator/README.md index 496a6d5bf5..068e2f60d7 100644 --- a/helm-charts/seldon-core-operator/README.md +++ b/helm-charts/seldon-core-operator/README.md @@ -37,6 +37,7 @@ helm install seldon-core-operator seldonio/seldon-core-operator --namespace seld | crd.create | bool | `true` | | | crd.forceV1 | bool | `false` | | | crd.forceV1beta1 | bool | `false` | | +| crd.annotations | map | `{}` | Annotations to add to the CRD | | credentials.gcs.gcsCredentialFileName | string | `"gcloud-application-credentials.json"` | | | credentials.s3.s3AccessKeyIDName | string | `"awsAccessKeyID"` | | | credentials.s3.s3SecretAccessKeyName | string | `"awsSecretAccessKey"` | | @@ -85,6 +86,7 @@ helm install seldon-core-operator seldonio/seldon-core-operator --namespace seld | manager.leaderElectionID | string | `"a33bd623.machinelearning.seldon.io"` | | | manager.memoryLimit | string | `"300Mi"` | | | manager.memoryRequest | string | `"200Mi"` | | +| manager.annotations | map | `{}` | Annotations to add to the deployment template spec | | managerCreateResources | bool | `false` | | | managerUserID | int | `8888` | | | namespaceOverride | string | `""` | | From cf48cf4058e40c94ac42101a65a4549a119e1035 Mon Sep 17 00:00:00 2001 From: Michael Meredith Date: Tue, 9 Nov 2021 22:21:03 -0500 Subject: [PATCH 7/8] remove deployment annotations --- .../templates/deployment_seldon-controller-manager.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml b/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml index 894f170a9e..ba328c9f93 100644 --- a/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml +++ b/helm-charts/seldon-core-operator/templates/deployment_seldon-controller-manager.yaml @@ -7,10 +7,6 @@ metadata: app.kubernetes.io/name: '{{ include "seldon.name" . }}' app.kubernetes.io/version: '{{ .Chart.Version }}' control-plane: seldon-controller-manager - {{with .Values.manager.annotations}} - annotations: - {{- toYaml . | nindent 4}} - {{- end }} name: seldon-controller-manager namespace: '{{ include "seldon.namespace" . }}' spec: From 24c85eea677d361e0e987fdf84155b4beb22d786 Mon Sep 17 00:00:00 2001 From: Michael Meredith Date: Tue, 9 Nov 2021 22:34:17 -0500 Subject: [PATCH 8/8] minor tweaks --- ...edefinition_seldondeployments.machinelearning.seldon.io.yaml | 2 +- ...finition_v1_seldondeployments.machinelearning.seldon.io.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml b/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml index 4d79331f74..be053812b6 100644 --- a/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml +++ b/helm-charts/seldon-core-operator/templates/customresourcedefinition_seldondeployments.machinelearning.seldon.io.yaml @@ -5,7 +5,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: '{{ include "seldon.namespace" . }}/seldon-serving-cert' - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.5.0 {{- with .Values.crd.annotations }} {{- toYaml . | nindent 4}} {{- end }} diff --git a/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml b/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml index 402bcaa9e2..f27ad36d35 100644 --- a/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml +++ b/helm-charts/seldon-core-operator/templates/customresourcedefinition_v1_seldondeployments.machinelearning.seldon.io.yaml @@ -5,7 +5,7 @@ kind: CustomResourceDefinition metadata: annotations: cert-manager.io/inject-ca-from: '{{ include "seldon.namespace" . }}/seldon-serving-cert' - controller-gen.kubebuilder.io/version: v0.4.1 + controller-gen.kubebuilder.io/version: v0.5.0 {{- with .Values.crd.annotations }} {{- toYaml . | nindent 4}} {{- end }}